2024-04-26T04:26:36.822 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-26T04:26:36.822 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --description upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} --name teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi --archive /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674565 --verbose -- /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674565/orig.config.yaml 2024-04-26T04:26:36.881 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-26T04:26:36.976 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674565 branch: squid description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' first_in_suite: false job_id: '7674565' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic cls tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: squid sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 8320 sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 10/32 suite: upgrade suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a targets: smithi156.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM2HXQRkDls+lC4fcS46mnWL3XBA8xna/tyo6rjNsgsOoG+yq0T9ZYBIUPBdqThsWnNy9o2Um+r879jCRbZGeHI= smithi171.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGcPaoiQ6uEEM2skH/wzALXnaue1QlScxv574JI4U9eglkXMB8nyMxVbjl/VlSOv1IF1UwMvXMMWK8+ACxW7Ud0= tasks: - pexec: all: - sudo dnf install runc -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-26_02:08:13 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-26T04:26:36.977 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa; will attempt to use it 2024-04-26T04:26:36.977 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks 2024-04-26T04:26:36.977 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-26T04:26:36.979 INFO:teuthology.task.internal:Checking packages... 2024-04-26T04:26:37.007 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a' 2024-04-26T04:26:37.007 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:26:37.007 INFO:teuthology.packaging:ref: None 2024-04-26T04:26:37.007 INFO:teuthology.packaging:tag: None 2024-04-26T04:26:37.007 INFO:teuthology.packaging:branch: squid 2024-04-26T04:26:37.008 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:26:37.008 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-04-26T04:26:37.333 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2447.gb22e2ebd 2024-04-26T04:26:37.334 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-26T04:26:37.341 INFO:teuthology.task.internal:no buildpackages task found 2024-04-26T04:26:37.341 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-26T04:26:37.355 INFO:teuthology.task.internal:Saving configuration 2024-04-26T04:26:37.379 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-26T04:26:37.384 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T04:26:37.408 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi156.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674565', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:21:28.743832', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM2HXQRkDls+lC4fcS46mnWL3XBA8xna/tyo6rjNsgsOoG+yq0T9ZYBIUPBdqThsWnNy9o2Um+r879jCRbZGeHI='} 2024-04-26T04:26:37.430 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi171.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674565', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:21:28.745736', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGcPaoiQ6uEEM2skH/wzALXnaue1QlScxv574JI4U9eglkXMB8nyMxVbjl/VlSOv1IF1UwMvXMMWK8+ACxW7Ud0='} 2024-04-26T04:26:37.430 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-26T04:26:37.436 INFO:teuthology.task.internal:roles: ubuntu@smithi156.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-26T04:26:37.436 INFO:teuthology.task.internal:roles: ubuntu@smithi171.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-26T04:26:37.436 INFO:teuthology.run_tasks:Running task console_log... 2024-04-26T04:26:37.507 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f8bb76dc0d0>, signals=[15]) 2024-04-26T04:26:37.508 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-26T04:26:37.514 INFO:teuthology.task.internal:Opening connections... 2024-04-26T04:26:37.514 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi156.front.sepia.ceph.com 2024-04-26T04:26:37.516 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi156.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:26:37.596 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi171.front.sepia.ceph.com 2024-04-26T04:26:37.597 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi171.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:26:37.672 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-26T04:26:37.678 DEBUG:teuthology.orchestra.run.smithi156:> uname -m 2024-04-26T04:26:37.696 INFO:teuthology.orchestra.run.smithi156.stdout:x86_64 2024-04-26T04:26:37.697 DEBUG:teuthology.orchestra.run.smithi156:> cat /etc/os-release 2024-04-26T04:26:37.754 INFO:teuthology.orchestra.run.smithi156.stdout:NAME="CentOS Stream" 2024-04-26T04:26:37.754 INFO:teuthology.orchestra.run.smithi156.stdout:VERSION="9" 2024-04-26T04:26:37.755 INFO:teuthology.orchestra.run.smithi156.stdout:ID="centos" 2024-04-26T04:26:37.755 INFO:teuthology.orchestra.run.smithi156.stdout:ID_LIKE="rhel fedora" 2024-04-26T04:26:37.755 INFO:teuthology.orchestra.run.smithi156.stdout:VERSION_ID="9" 2024-04-26T04:26:37.755 INFO:teuthology.orchestra.run.smithi156.stdout:PLATFORM_ID="platform:el9" 2024-04-26T04:26:37.755 INFO:teuthology.orchestra.run.smithi156.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-26T04:26:37.755 INFO:teuthology.orchestra.run.smithi156.stdout:ANSI_COLOR="0;31" 2024-04-26T04:26:37.755 INFO:teuthology.orchestra.run.smithi156.stdout:LOGO="fedora-logo-icon" 2024-04-26T04:26:37.755 INFO:teuthology.orchestra.run.smithi156.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-26T04:26:37.755 INFO:teuthology.orchestra.run.smithi156.stdout:HOME_URL="https://centos.org/" 2024-04-26T04:26:37.755 INFO:teuthology.orchestra.run.smithi156.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-26T04:26:37.755 INFO:teuthology.orchestra.run.smithi156.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-26T04:26:37.755 INFO:teuthology.orchestra.run.smithi156.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-26T04:26:37.756 INFO:teuthology.lock.ops:Updating smithi156.front.sepia.ceph.com on lock server 2024-04-26T04:26:37.781 DEBUG:teuthology.orchestra.run.smithi171:> uname -m 2024-04-26T04:26:37.799 INFO:teuthology.orchestra.run.smithi171.stdout:x86_64 2024-04-26T04:26:37.799 DEBUG:teuthology.orchestra.run.smithi171:> cat /etc/os-release 2024-04-26T04:26:37.856 INFO:teuthology.orchestra.run.smithi171.stdout:NAME="CentOS Stream" 2024-04-26T04:26:37.856 INFO:teuthology.orchestra.run.smithi171.stdout:VERSION="9" 2024-04-26T04:26:37.856 INFO:teuthology.orchestra.run.smithi171.stdout:ID="centos" 2024-04-26T04:26:37.856 INFO:teuthology.orchestra.run.smithi171.stdout:ID_LIKE="rhel fedora" 2024-04-26T04:26:37.856 INFO:teuthology.orchestra.run.smithi171.stdout:VERSION_ID="9" 2024-04-26T04:26:37.856 INFO:teuthology.orchestra.run.smithi171.stdout:PLATFORM_ID="platform:el9" 2024-04-26T04:26:37.856 INFO:teuthology.orchestra.run.smithi171.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-26T04:26:37.856 INFO:teuthology.orchestra.run.smithi171.stdout:ANSI_COLOR="0;31" 2024-04-26T04:26:37.856 INFO:teuthology.orchestra.run.smithi171.stdout:LOGO="fedora-logo-icon" 2024-04-26T04:26:37.856 INFO:teuthology.orchestra.run.smithi171.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-26T04:26:37.857 INFO:teuthology.orchestra.run.smithi171.stdout:HOME_URL="https://centos.org/" 2024-04-26T04:26:37.857 INFO:teuthology.orchestra.run.smithi171.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-26T04:26:37.857 INFO:teuthology.orchestra.run.smithi171.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-26T04:26:37.857 INFO:teuthology.orchestra.run.smithi171.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-26T04:26:37.857 INFO:teuthology.lock.ops:Updating smithi171.front.sepia.ceph.com on lock server 2024-04-26T04:26:37.881 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-26T04:26:37.889 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-26T04:26:37.894 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-26T04:26:37.894 DEBUG:teuthology.orchestra.run.smithi156:> test '!' -e /home/ubuntu/cephtest 2024-04-26T04:26:37.896 DEBUG:teuthology.orchestra.run.smithi171:> test '!' -e /home/ubuntu/cephtest 2024-04-26T04:26:37.912 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-26T04:26:37.918 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-26T04:26:37.918 DEBUG:teuthology.orchestra.run.smithi156:> test -z $(ls -A /var/lib/ceph) 2024-04-26T04:26:37.953 DEBUG:teuthology.orchestra.run.smithi171:> test -z $(ls -A /var/lib/ceph) 2024-04-26T04:26:38.057 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-26T04:26:38.109 INFO:teuthology.run_tasks:Running task kernel... 2024-04-26T04:26:38.124 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-26T04:26:38.124 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-26T04:26:38.124 DEBUG:teuthology.orchestra.run.smithi156:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-26T04:26:38.125 DEBUG:teuthology.orchestra.run.smithi171:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-26T04:26:38.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:26:38.142 DEBUG:teuthology.orchestra.run.smithi171:> uname -r 2024-04-26T04:26:38.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:26:38.144 DEBUG:teuthology.orchestra.run.smithi156:> uname -r 2024-04-26T04:26:38.199 INFO:teuthology.orchestra.run.smithi171.stdout:5.14.0-437.el9.x86_64 2024-04-26T04:26:38.200 INFO:teuthology.task.kernel:Running kernel on smithi171: 5.14.0-437.el9.x86_64 2024-04-26T04:26:38.200 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum install -y kernel 2024-04-26T04:26:38.201 INFO:teuthology.orchestra.run.smithi156.stdout:5.14.0-437.el9.x86_64 2024-04-26T04:26:38.201 INFO:teuthology.task.kernel:Running kernel on smithi156: 5.14.0-437.el9.x86_64 2024-04-26T04:26:38.201 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum install -y kernel 2024-04-26T04:26:40.498 INFO:teuthology.orchestra.run.smithi171.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.0 MB 00:00 2024-04-26T04:26:42.408 INFO:teuthology.orchestra.run.smithi171.stdout:CentOS Stream 9 - AppStream 22 MB/s | 19 MB 00:00 2024-04-26T04:26:47.206 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS Stream 9 - BaseOS 1.1 MB/s | 8.0 MB 00:07 2024-04-26T04:26:47.267 INFO:teuthology.orchestra.run.smithi171.stdout:CentOS Stream 9 - CRB 4.7 MB/s | 6.2 MB 00:01 2024-04-26T04:26:48.956 INFO:teuthology.orchestra.run.smithi171.stdout:CentOS Stream 9 - Extras packages 48 kB/s | 16 kB 00:00 2024-04-26T04:26:49.658 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS Stream 9 - AppStream 14 MB/s | 19 MB 00:01 2024-04-26T04:26:50.866 INFO:teuthology.orchestra.run.smithi171.stdout:Extra Packages for Enterprise Linux 12 MB/s | 21 MB 00:01 2024-04-26T04:26:54.134 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS Stream 9 - CRB 7.6 MB/s | 6.2 MB 00:00 2024-04-26T04:26:55.699 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS Stream 9 - Extras packages 39 kB/s | 16 kB 00:00 2024-04-26T04:26:55.857 INFO:teuthology.orchestra.run.smithi171.stdout:lab-extras 14 kB/s | 1.7 kB 00:00 2024-04-26T04:26:56.043 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:01 ago on Fri 26 Apr 2024 04:26:55 AM UTC. 2024-04-26T04:26:57.200 INFO:teuthology.orchestra.run.smithi156.stdout:Extra Packages for Enterprise Linux 15 MB/s | 21 MB 00:01 2024-04-26T04:26:57.788 INFO:teuthology.orchestra.run.smithi171.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-26T04:26:57.789 INFO:teuthology.orchestra.run.smithi171.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-26T04:26:57.874 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:26:57.875 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-04-26T04:26:57.876 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:26:57.982 DEBUG:teuthology.orchestra.run.smithi171:> echo no | sudo yum reinstall kernel || true 2024-04-26T04:26:58.486 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 04:26:55 AM UTC. 2024-04-26T04:26:58.614 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:26:58.636 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:26:58.636 INFO:teuthology.orchestra.run.smithi171.stdout: Package Architecture Version Repository Size 2024-04-26T04:26:58.636 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:26:58.636 INFO:teuthology.orchestra.run.smithi171.stdout:Reinstalling: 2024-04-26T04:26:58.636 INFO:teuthology.orchestra.run.smithi171.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:26:58.636 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:26:58.637 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:26:58.637 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:26:58.637 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:26:58.637 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 255 k 2024-04-26T04:26:58.637 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 0 2024-04-26T04:26:58.637 INFO:teuthology.orchestra.run.smithi171.stderr:Operation aborted. 2024-04-26T04:26:58.700 INFO:teuthology.orchestra.run.smithi171.stdout:Is this ok [y/N]: 2024-04-26T04:26:58.700 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum reinstall -y kernel || true 2024-04-26T04:26:59.193 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:04 ago on Fri 26 Apr 2024 04:26:55 AM UTC. 2024-04-26T04:26:59.311 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:26:59.311 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:26:59.311 INFO:teuthology.orchestra.run.smithi171.stdout: Package Architecture Version Repository Size 2024-04-26T04:26:59.311 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:26:59.311 INFO:teuthology.orchestra.run.smithi171.stdout:Reinstalling: 2024-04-26T04:26:59.312 INFO:teuthology.orchestra.run.smithi171.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:26:59.312 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:26:59.312 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:26:59.312 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:26:59.312 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:26:59.312 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 255 k 2024-04-26T04:26:59.312 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 0 2024-04-26T04:26:59.312 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-04-26T04:26:59.518 INFO:teuthology.orchestra.run.smithi171.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.7 MB/s | 255 kB 00:00 2024-04-26T04:26:59.518 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:26:59.518 INFO:teuthology.orchestra.run.smithi171.stdout:Total 1.2 MB/s | 255 kB 00:00 2024-04-26T04:26:59.538 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-04-26T04:26:59.722 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-04-26T04:26:59.723 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-04-26T04:26:59.728 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-04-26T04:26:59.728 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-04-26T04:26:59.863 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-04-26T04:26:59.936 INFO:teuthology.orchestra.run.smithi171.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:26:59.936 INFO:teuthology.orchestra.run.smithi171.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:27:00.583 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:27:00.584 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:27:01.238 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:27:01.238 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:27:01.238 INFO:teuthology.orchestra.run.smithi171.stdout:Reinstalled: 2024-04-26T04:27:01.238 INFO:teuthology.orchestra.run.smithi171.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-26T04:27:01.238 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:27:01.238 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:27:01.405 DEBUG:teuthology.orchestra.run.smithi171:> rpm -q kernel | sort -rV | head -n 1 2024-04-26T04:27:01.451 INFO:teuthology.orchestra.run.smithi171.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-26T04:27:01.452 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-26T04:27:01.452 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-26T04:27:01.452 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-26T04:27:01.452 DEBUG:teuthology.orchestra.run.smithi171:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-26T04:27:01.485 INFO:teuthology.orchestra.run.smithi171.stdout:ttyS1 2024-04-26T04:27:01.505 DEBUG:teuthology.parallel:result is None 2024-04-26T04:27:02.250 INFO:teuthology.orchestra.run.smithi156.stdout:lab-extras 12 kB/s | 1.7 kB 00:00 2024-04-26T04:27:04.140 INFO:teuthology.orchestra.run.smithi156.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-26T04:27:04.140 INFO:teuthology.orchestra.run.smithi156.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-26T04:27:04.221 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:27:04.222 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-04-26T04:27:04.222 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:27:04.338 DEBUG:teuthology.orchestra.run.smithi156:> echo no | sudo yum reinstall kernel || true 2024-04-26T04:27:04.842 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:02 ago on Fri 26 Apr 2024 04:27:02 AM UTC. 2024-04-26T04:27:04.962 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:27:04.975 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:27:04.975 INFO:teuthology.orchestra.run.smithi156.stdout: Package Architecture Version Repository Size 2024-04-26T04:27:04.975 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:27:04.975 INFO:teuthology.orchestra.run.smithi156.stdout:Reinstalling: 2024-04-26T04:27:04.975 INFO:teuthology.orchestra.run.smithi156.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:27:04.975 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:27:04.975 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:27:04.975 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:27:04.975 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:27:04.976 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 255 k 2024-04-26T04:27:04.976 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 0 2024-04-26T04:27:04.976 INFO:teuthology.orchestra.run.smithi156.stderr:Operation aborted. 2024-04-26T04:27:05.044 INFO:teuthology.orchestra.run.smithi156.stdout:Is this ok [y/N]: 2024-04-26T04:27:05.045 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum reinstall -y kernel || true 2024-04-26T04:27:05.539 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 04:27:02 AM UTC. 2024-04-26T04:27:05.654 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:27:05.655 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:27:05.655 INFO:teuthology.orchestra.run.smithi156.stdout: Package Architecture Version Repository Size 2024-04-26T04:27:05.655 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:27:05.655 INFO:teuthology.orchestra.run.smithi156.stdout:Reinstalling: 2024-04-26T04:27:05.655 INFO:teuthology.orchestra.run.smithi156.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:27:05.655 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:27:05.655 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:27:05.655 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:27:05.655 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:27:05.656 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 255 k 2024-04-26T04:27:05.656 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 0 2024-04-26T04:27:05.656 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-04-26T04:27:05.877 INFO:teuthology.orchestra.run.smithi156.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.7 MB/s | 255 kB 00:00 2024-04-26T04:27:05.877 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:27:05.878 INFO:teuthology.orchestra.run.smithi156.stdout:Total 1.1 MB/s | 255 kB 00:00 2024-04-26T04:27:05.897 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-04-26T04:27:06.092 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-04-26T04:27:06.092 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-04-26T04:27:06.097 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-04-26T04:27:06.097 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-04-26T04:27:06.247 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-04-26T04:27:06.328 INFO:teuthology.orchestra.run.smithi156.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:27:06.328 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:27:06.989 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:27:06.989 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:27:07.666 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:27:07.666 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:27:07.666 INFO:teuthology.orchestra.run.smithi156.stdout:Reinstalled: 2024-04-26T04:27:07.666 INFO:teuthology.orchestra.run.smithi156.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-26T04:27:07.666 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:27:07.666 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:27:07.865 DEBUG:teuthology.orchestra.run.smithi156:> rpm -q kernel | sort -rV | head -n 1 2024-04-26T04:27:07.912 INFO:teuthology.orchestra.run.smithi156.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-26T04:27:07.912 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-26T04:27:07.912 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-26T04:27:07.912 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-26T04:27:07.913 DEBUG:teuthology.orchestra.run.smithi156:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-26T04:27:07.946 INFO:teuthology.orchestra.run.smithi156.stdout:ttyS1 2024-04-26T04:27:07.972 DEBUG:teuthology.parallel:result is None 2024-04-26T04:27:07.973 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-26T04:27:07.980 INFO:teuthology.task.internal:Creating test directory... 2024-04-26T04:27:07.980 DEBUG:teuthology.orchestra.run.smithi156:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-26T04:27:08.016 DEBUG:teuthology.orchestra.run.smithi171:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-26T04:27:08.033 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-26T04:27:08.041 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-26T04:27:08.049 INFO:teuthology.task.internal:Creating archive directory... 2024-04-26T04:27:08.050 DEBUG:teuthology.orchestra.run.smithi156:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-26T04:27:08.070 DEBUG:teuthology.orchestra.run.smithi171:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-26T04:27:08.183 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-26T04:27:08.191 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-26T04:27:08.191 DEBUG:teuthology.orchestra.run.smithi156:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-26T04:27:08.194 DEBUG:teuthology.orchestra.run.smithi171:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-26T04:27:08.227 INFO:teuthology.orchestra.run.smithi171.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:27:08.227 INFO:teuthology.orchestra.run.smithi156.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:27:08.236 INFO:teuthology.orchestra.run.smithi156.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:27:08.238 INFO:teuthology.orchestra.run.smithi171.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:27:08.239 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-26T04:27:08.248 INFO:teuthology.task.internal:Configuring sudo... 2024-04-26T04:27:08.248 DEBUG:teuthology.orchestra.run.smithi156:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-26T04:27:08.280 DEBUG:teuthology.orchestra.run.smithi171:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-26T04:27:08.307 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-26T04:27:08.316 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-26T04:27:08.317 DEBUG:teuthology.orchestra.run.smithi156:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-26T04:27:08.345 DEBUG:teuthology.orchestra.run.smithi171:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-26T04:27:08.363 DEBUG:teuthology.orchestra.run.smithi156:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:27:08.442 DEBUG:teuthology.orchestra.run.smithi156:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:27:08.504 DEBUG:teuthology.orchestra.run.smithi156:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:27:08.543 DEBUG:teuthology.orchestra.run.smithi156:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:27:08.587 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-04-26T04:27:08.587 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-26T04:27:08.650 DEBUG:teuthology.orchestra.run.smithi171:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:27:08.695 DEBUG:teuthology.orchestra.run.smithi171:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:27:08.744 DEBUG:teuthology.orchestra.run.smithi171:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:27:08.828 DEBUG:teuthology.orchestra.run.smithi171:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:27:08.868 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-04-26T04:27:08.868 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-26T04:27:08.934 DEBUG:teuthology.orchestra.run.smithi156:> sudo service rsyslog restart 2024-04-26T04:27:08.936 DEBUG:teuthology.orchestra.run.smithi171:> sudo service rsyslog restart 2024-04-26T04:27:08.968 INFO:teuthology.orchestra.run.smithi156.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T04:27:09.006 INFO:teuthology.orchestra.run.smithi171.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T04:27:09.378 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-26T04:27:09.387 INFO:teuthology.task.internal:Starting timer... 2024-04-26T04:27:09.387 INFO:teuthology.run_tasks:Running task pcp... 2024-04-26T04:27:09.401 INFO:teuthology.run_tasks:Running task selinux... 2024-04-26T04:27:09.416 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-26T04:27:09.416 DEBUG:teuthology.orchestra.run.smithi156:> sudo service auditd rotate 2024-04-26T04:27:09.471 INFO:teuthology.orchestra.run.smithi156.stdout:Rotating logs: 2024-04-26T04:27:09.473 DEBUG:teuthology.orchestra.run.smithi171:> sudo service auditd rotate 2024-04-26T04:27:09.551 INFO:teuthology.orchestra.run.smithi171.stdout:Rotating logs: 2024-04-26T04:27:09.553 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-26T04:27:09.553 DEBUG:teuthology.orchestra.run.smithi156:> /usr/sbin/getenforce 2024-04-26T04:27:09.578 INFO:teuthology.orchestra.run.smithi156.stdout:Permissive 2024-04-26T04:27:09.578 DEBUG:teuthology.orchestra.run.smithi171:> /usr/sbin/getenforce 2024-04-26T04:27:09.597 INFO:teuthology.orchestra.run.smithi171.stdout:Permissive 2024-04-26T04:27:09.597 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi156.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi171.front.sepia.ceph.com': 'permissive'} 2024-04-26T04:27:09.597 DEBUG:teuthology.orchestra.run.smithi156:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T04:27:09.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:27:09.643 DEBUG:teuthology.orchestra.run.smithi171:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T04:27:09.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:27:09.667 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-26T04:27:09.667 DEBUG:teuthology.orchestra.run.smithi156:> sudo /usr/sbin/setenforce permissive 2024-04-26T04:27:09.714 DEBUG:teuthology.orchestra.run.smithi171:> sudo /usr/sbin/setenforce permissive 2024-04-26T04:27:09.738 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-26T04:27:09.749 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-26T04:27:09.754 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-26T04:27:09.882 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-26T04:27:09.898 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-26T04:27:09.900 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi156.front.sepia.ceph.com,smithi171.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-26T04:30:41.874 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi156.front.sepia.ceph.com'), Remote(name='ubuntu@smithi171.front.sepia.ceph.com')] 2024-04-26T04:30:41.875 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi156.front.sepia.ceph.com' 2024-04-26T04:30:41.876 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi156.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:30:41.953 DEBUG:teuthology.orchestra.run.smithi156:> true 2024-04-26T04:30:42.042 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi156.front.sepia.ceph.com' 2024-04-26T04:30:42.042 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi171.front.sepia.ceph.com' 2024-04-26T04:30:42.043 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi171.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:30:42.117 DEBUG:teuthology.orchestra.run.smithi171:> true 2024-04-26T04:30:42.196 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi171.front.sepia.ceph.com' 2024-04-26T04:30:42.196 INFO:teuthology.run_tasks:Running task clock... 2024-04-26T04:30:42.207 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-26T04:30:42.207 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-26T04:30:42.208 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T04:30:42.210 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-26T04:30:42.210 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T04:30:42.238 INFO:teuthology.orchestra.run.smithi156.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-26T04:30:42.252 INFO:teuthology.orchestra.run.smithi156.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-26T04:30:42.266 INFO:teuthology.orchestra.run.smithi171.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-26T04:30:42.277 INFO:teuthology.orchestra.run.smithi156.stderr:sudo: ntpd: command not found 2024-04-26T04:30:42.280 INFO:teuthology.orchestra.run.smithi171.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-26T04:30:42.286 INFO:teuthology.orchestra.run.smithi156.stdout:506 Cannot talk to daemon 2024-04-26T04:30:42.300 INFO:teuthology.orchestra.run.smithi156.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-26T04:30:42.304 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: ntpd: command not found 2024-04-26T04:30:42.313 INFO:teuthology.orchestra.run.smithi156.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-26T04:30:42.314 INFO:teuthology.orchestra.run.smithi171.stdout:506 Cannot talk to daemon 2024-04-26T04:30:42.328 INFO:teuthology.orchestra.run.smithi171.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-26T04:30:42.341 INFO:teuthology.orchestra.run.smithi171.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-26T04:30:42.381 INFO:teuthology.orchestra.run.smithi156.stderr:bash: line 1: ntpq: command not found 2024-04-26T04:30:42.384 INFO:teuthology.orchestra.run.smithi156.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T04:30:42.385 INFO:teuthology.orchestra.run.smithi156.stdout:=============================================================================== 2024-04-26T04:30:42.385 INFO:teuthology.orchestra.run.smithi156.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:30:42.385 INFO:teuthology.orchestra.run.smithi156.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:30:42.385 INFO:teuthology.orchestra.run.smithi156.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:30:42.385 INFO:teuthology.orchestra.run.smithi156.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:30:42.394 INFO:teuthology.orchestra.run.smithi171.stderr:bash: line 1: ntpq: command not found 2024-04-26T04:30:42.398 INFO:teuthology.orchestra.run.smithi171.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T04:30:42.398 INFO:teuthology.orchestra.run.smithi171.stdout:=============================================================================== 2024-04-26T04:30:42.398 INFO:teuthology.orchestra.run.smithi171.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:30:42.398 INFO:teuthology.orchestra.run.smithi171.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:30:42.398 INFO:teuthology.orchestra.run.smithi171.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:30:42.398 INFO:teuthology.orchestra.run.smithi171.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:30:42.399 INFO:teuthology.run_tasks:Running task pexec... 2024-04-26T04:30:42.408 INFO:teuthology.task.pexec:Executing custom commands... 2024-04-26T04:30:42.409 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi156.front.sepia.ceph.com 2024-04-26T04:30:42.409 DEBUG:teuthology.orchestra.run.smithi156:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-04-26T04:30:42.409 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi171.front.sepia.ceph.com 2024-04-26T04:30:42.409 DEBUG:teuthology.orchestra.run.smithi171:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-04-26T04:30:42.954 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:41 ago on Fri 26 Apr 2024 04:30:01 AM UTC. 2024-04-26T04:30:42.966 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:41 ago on Fri 26 Apr 2024 04:30:01 AM UTC. 2024-04-26T04:30:43.071 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:30:43.072 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:30:43.072 INFO:teuthology.orchestra.run.smithi156.stdout: Package Architecture Version Repository Size 2024-04-26T04:30:43.072 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:30:43.072 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-04-26T04:30:43.072 INFO:teuthology.orchestra.run.smithi156.stdout: runc x86_64 4:1.1.12-2.el9 appstream 3.1 M 2024-04-26T04:30:43.072 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:30:43.072 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:30:43.072 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:30:43.072 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2024-04-26T04:30:43.072 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:30:43.073 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 3.1 M 2024-04-26T04:30:43.073 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 9.8 M 2024-04-26T04:30:43.073 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-04-26T04:30:43.084 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:30:43.085 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:30:43.085 INFO:teuthology.orchestra.run.smithi171.stdout: Package Architecture Version Repository Size 2024-04-26T04:30:43.085 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:30:43.085 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-04-26T04:30:43.085 INFO:teuthology.orchestra.run.smithi171.stdout: runc x86_64 4:1.1.12-2.el9 appstream 3.1 M 2024-04-26T04:30:43.085 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:30:43.085 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:30:43.085 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:30:43.085 INFO:teuthology.orchestra.run.smithi171.stdout:Install 1 Package 2024-04-26T04:30:43.085 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:30:43.085 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 3.1 M 2024-04-26T04:30:43.085 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 9.8 M 2024-04-26T04:30:43.085 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-04-26T04:30:43.392 INFO:teuthology.orchestra.run.smithi156.stdout:runc-1.1.12-2.el9.x86_64.rpm 12 MB/s | 3.1 MB 00:00 2024-04-26T04:30:43.393 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:30:43.393 INFO:teuthology.orchestra.run.smithi156.stdout:Total 9.8 MB/s | 3.1 MB 00:00 2024-04-26T04:30:43.430 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-04-26T04:30:43.436 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-04-26T04:30:43.436 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-04-26T04:30:43.518 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-04-26T04:30:43.518 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-04-26T04:30:43.728 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-04-26T04:30:43.834 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-26T04:30:44.320 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-26T04:30:44.598 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-26T04:30:44.598 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:30:44.598 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-04-26T04:30:44.599 INFO:teuthology.orchestra.run.smithi156.stdout: runc-4:1.1.12-2.el9.x86_64 2024-04-26T04:30:44.599 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:30:44.599 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:30:44.775 INFO:teuthology.orchestra.run.smithi171.stdout:runc-1.1.12-2.el9.x86_64.rpm 1.9 MB/s | 3.1 MB 00:01 2024-04-26T04:30:44.776 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:30:44.777 INFO:teuthology.orchestra.run.smithi171.stdout:Total 1.8 MB/s | 3.1 MB 00:01 2024-04-26T04:30:44.815 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-04-26T04:30:44.822 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-04-26T04:30:44.822 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-04-26T04:30:44.855 DEBUG:teuthology.parallel:result is None 2024-04-26T04:30:44.908 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-04-26T04:30:44.908 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-04-26T04:30:45.132 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-04-26T04:30:45.234 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-26T04:30:45.672 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-26T04:30:45.949 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-26T04:30:45.949 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:30:45.949 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-04-26T04:30:45.949 INFO:teuthology.orchestra.run.smithi171.stdout: runc-4:1.1.12-2.el9.x86_64 2024-04-26T04:30:45.949 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:30:45.950 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:30:46.175 DEBUG:teuthology.parallel:result is None 2024-04-26T04:30:46.176 INFO:teuthology.run_tasks:Running task install... 2024-04-26T04:30:46.185 DEBUG:teuthology.task.install:project ceph 2024-04-26T04:30:46.185 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a'}} 2024-04-26T04:30:46.185 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a'} 2024-04-26T04:30:46.185 INFO:teuthology.task.install:Using flavor: default 2024-04-26T04:30:46.194 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-04-26T04:30:46.194 INFO:teuthology.task.install:extra packages: [] 2024-04-26T04:30:46.195 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a', 'tag': None, 'wait_for_package': False} 2024-04-26T04:30:46.195 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:30:46.195 INFO:teuthology.packaging:ref: None 2024-04-26T04:30:46.195 INFO:teuthology.packaging:tag: None 2024-04-26T04:30:46.195 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:30:46.195 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:30:46.195 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:30:46.198 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a', 'tag': None, 'wait_for_package': False} 2024-04-26T04:30:46.198 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:30:46.198 INFO:teuthology.packaging:ref: None 2024-04-26T04:30:46.198 INFO:teuthology.packaging:tag: None 2024-04-26T04:30:46.198 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:30:46.198 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:30:46.198 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:30:46.447 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/ 2024-04-26T04:30:46.447 INFO:teuthology.task.install.rpm:Package version is 17.2.7-904.gd8c6b0a3 2024-04-26T04:30:46.452 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/ 2024-04-26T04:30:46.452 INFO:teuthology.task.install.rpm:Package version is 17.2.7-904.gd8c6b0a3 2024-04-26T04:30:46.558 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-26T04:30:46.558 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-04-26T04:30:46.558 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-26T04:30:46.566 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-26T04:30:46.566 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-04-26T04:30:46.566 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-26T04:30:46.583 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-26T04:30:46.583 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:30:46.583 INFO:teuthology.packaging:ref: None 2024-04-26T04:30:46.583 INFO:teuthology.packaging:tag: None 2024-04-26T04:30:46.583 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:30:46.583 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:30:46.583 DEBUG:teuthology.orchestra.run.smithi171:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-26T04:30:46.594 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-26T04:30:46.594 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:30:46.594 INFO:teuthology.packaging:ref: None 2024-04-26T04:30:46.594 INFO:teuthology.packaging:tag: None 2024-04-26T04:30:46.594 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:30:46.594 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:30:46.594 DEBUG:teuthology.orchestra.run.smithi156:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-26T04:30:46.647 DEBUG:teuthology.orchestra.run.smithi171:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-26T04:30:46.661 DEBUG:teuthology.orchestra.run.smithi156:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-26T04:30:46.721 DEBUG:teuthology.orchestra.run.smithi171:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-26T04:30:46.738 DEBUG:teuthology.orchestra.run.smithi156:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-26T04:30:46.786 INFO:teuthology.orchestra.run.smithi171.stdout:check_obsoletes = 1 2024-04-26T04:30:46.787 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum clean all 2024-04-26T04:30:46.803 INFO:teuthology.orchestra.run.smithi156.stdout:check_obsoletes = 1 2024-04-26T04:30:46.805 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum clean all 2024-04-26T04:30:47.019 INFO:teuthology.orchestra.run.smithi171.stdout:48 files removed 2024-04-26T04:30:47.036 INFO:teuthology.orchestra.run.smithi156.stdout:48 files removed 2024-04-26T04:30:47.048 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-radosgw 2024-04-26T04:30:47.064 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-radosgw 2024-04-26T04:30:47.738 INFO:teuthology.orchestra.run.smithi171.stdout:ceph packages for x86_64 183 kB/s | 77 kB 00:00 2024-04-26T04:30:47.740 INFO:teuthology.orchestra.run.smithi156.stdout:ceph packages for x86_64 188 kB/s | 77 kB 00:00 2024-04-26T04:30:48.157 INFO:teuthology.orchestra.run.smithi156.stdout:ceph noarch packages 29 kB/s | 11 kB 00:00 2024-04-26T04:30:48.201 INFO:teuthology.orchestra.run.smithi171.stdout:ceph noarch packages 25 kB/s | 11 kB 00:00 2024-04-26T04:30:48.466 INFO:teuthology.orchestra.run.smithi156.stdout:ceph source packages 6.5 kB/s | 1.8 kB 00:00 2024-04-26T04:30:48.622 INFO:teuthology.orchestra.run.smithi171.stdout:ceph source packages 4.6 kB/s | 1.8 kB 00:00 2024-04-26T04:30:49.961 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS Stream 9 - BaseOS 5.5 MB/s | 8.0 MB 00:01 2024-04-26T04:30:51.045 INFO:teuthology.orchestra.run.smithi171.stdout:CentOS Stream 9 - BaseOS 3.4 MB/s | 8.0 MB 00:02 2024-04-26T04:30:52.338 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS Stream 9 - AppStream 15 MB/s | 19 MB 00:01 2024-04-26T04:30:55.253 INFO:teuthology.orchestra.run.smithi171.stdout:CentOS Stream 9 - AppStream 6.1 MB/s | 19 MB 00:03 2024-04-26T04:30:56.830 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS Stream 9 - CRB 9.6 MB/s | 6.2 MB 00:00 2024-04-26T04:30:58.353 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS Stream 9 - Extras packages 55 kB/s | 16 kB 00:00 2024-04-26T04:31:01.304 INFO:teuthology.orchestra.run.smithi171.stdout:CentOS Stream 9 - CRB 2.8 MB/s | 6.2 MB 00:02 2024-04-26T04:31:03.504 INFO:teuthology.orchestra.run.smithi156.stdout:Extra Packages for Enterprise Linux 4.2 MB/s | 21 MB 00:05 2024-04-26T04:31:03.876 INFO:teuthology.orchestra.run.smithi171.stdout:CentOS Stream 9 - Extras packages 12 kB/s | 16 kB 00:01 2024-04-26T04:31:04.895 INFO:teuthology.orchestra.run.smithi171.stdout:Extra Packages for Enterprise Linux 23 MB/s | 21 MB 00:00 2024-04-26T04:31:08.537 INFO:teuthology.orchestra.run.smithi156.stdout:lab-extras 14 kB/s | 1.7 kB 00:00 2024-04-26T04:31:10.178 INFO:teuthology.orchestra.run.smithi171.stdout:lab-extras 16 kB/s | 1.7 kB 00:00 2024-04-26T04:31:10.673 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:31:10.675 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:31:10.675 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-04-26T04:31:10.675 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:31:10.676 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-04-26T04:31:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-radosgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 14 M 2024-04-26T04:31:10.676 INFO:teuthology.orchestra.run.smithi156.stdout:Upgrading: 2024-04-26T04:31:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: librados2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.1 M 2024-04-26T04:31:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: librbd1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:31:10.676 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2024-04-26T04:31:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-base x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 5.0 M 2024-04-26T04:31:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 22 M 2024-04-26T04:31:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-selinux x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:31:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-26T04:31:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: libcephfs2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 637 k 2024-04-26T04:31:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-26T04:31:10.676 INFO:teuthology.orchestra.run.smithi156.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-26T04:31:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-26T04:31:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: libradosstriper1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 457 k 2024-04-26T04:31:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-26T04:31:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: librgw2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 4.9 M 2024-04-26T04:31:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-26T04:31:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-26T04:31:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: python3-ceph-argparse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 45 k 2024-04-26T04:31:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: python3-ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 117 k 2024-04-26T04:31:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cephfs x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 156 k 2024-04-26T04:31:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-26T04:31:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rados x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 320 k 2024-04-26T04:31:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 293 k 2024-04-26T04:31:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 100 k 2024-04-26T04:31:10.677 INFO:teuthology.orchestra.run.smithi156.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-26T04:31:10.678 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:31:10.678 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:31:10.678 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:31:10.678 INFO:teuthology.orchestra.run.smithi156.stdout:Install 22 Packages 2024-04-26T04:31:10.678 INFO:teuthology.orchestra.run.smithi156.stdout:Upgrade 2 Packages 2024-04-26T04:31:10.678 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:31:10.678 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 58 M 2024-04-26T04:31:10.678 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-04-26T04:31:12.088 INFO:teuthology.orchestra.run.smithi156.stdout:(1/24): ceph-radosgw-17.2.7-904.gd8c6b0a3.el9.x 12 MB/s | 14 MB 00:01 2024-04-26T04:31:12.130 INFO:teuthology.orchestra.run.smithi156.stdout:(2/24): ceph-selinux-17.2.7-904.gd8c6b0a3.el9.x 608 kB/s | 25 kB 00:00 2024-04-26T04:31:12.197 INFO:teuthology.orchestra.run.smithi156.stdout:(3/24): libcephfs2-17.2.7-904.gd8c6b0a3.el9.x86 9.3 MB/s | 637 kB 00:00 2024-04-26T04:31:12.256 INFO:teuthology.orchestra.run.smithi156.stdout:(4/24): libradosstriper1-17.2.7-904.gd8c6b0a3.e 7.6 MB/s | 457 kB 00:00 2024-04-26T04:31:12.340 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:31:12.342 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:31:12.342 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-04-26T04:31:12.342 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:31:12.343 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-04-26T04:31:12.343 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-radosgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 14 M 2024-04-26T04:31:12.343 INFO:teuthology.orchestra.run.smithi171.stdout:Upgrading: 2024-04-26T04:31:12.343 INFO:teuthology.orchestra.run.smithi171.stdout: librados2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.1 M 2024-04-26T04:31:12.343 INFO:teuthology.orchestra.run.smithi171.stdout: librbd1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:31:12.343 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2024-04-26T04:31:12.343 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-base x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 5.0 M 2024-04-26T04:31:12.343 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 22 M 2024-04-26T04:31:12.343 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-selinux x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:31:12.343 INFO:teuthology.orchestra.run.smithi171.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-26T04:31:12.343 INFO:teuthology.orchestra.run.smithi171.stdout: libcephfs2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 637 k 2024-04-26T04:31:12.343 INFO:teuthology.orchestra.run.smithi171.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-26T04:31:12.343 INFO:teuthology.orchestra.run.smithi171.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-26T04:31:12.344 INFO:teuthology.orchestra.run.smithi171.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-26T04:31:12.344 INFO:teuthology.orchestra.run.smithi171.stdout: libradosstriper1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 457 k 2024-04-26T04:31:12.344 INFO:teuthology.orchestra.run.smithi171.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-26T04:31:12.344 INFO:teuthology.orchestra.run.smithi171.stdout: librgw2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 4.9 M 2024-04-26T04:31:12.344 INFO:teuthology.orchestra.run.smithi171.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-26T04:31:12.344 INFO:teuthology.orchestra.run.smithi171.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-26T04:31:12.344 INFO:teuthology.orchestra.run.smithi171.stdout: python3-ceph-argparse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 45 k 2024-04-26T04:31:12.344 INFO:teuthology.orchestra.run.smithi171.stdout: python3-ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 117 k 2024-04-26T04:31:12.344 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cephfs x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 156 k 2024-04-26T04:31:12.344 INFO:teuthology.orchestra.run.smithi171.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-26T04:31:12.344 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rados x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 320 k 2024-04-26T04:31:12.344 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 293 k 2024-04-26T04:31:12.344 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 100 k 2024-04-26T04:31:12.344 INFO:teuthology.orchestra.run.smithi171.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-26T04:31:12.345 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:31:12.345 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:31:12.345 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:31:12.345 INFO:teuthology.orchestra.run.smithi171.stdout:Install 22 Packages 2024-04-26T04:31:12.345 INFO:teuthology.orchestra.run.smithi171.stdout:Upgrade 2 Packages 2024-04-26T04:31:12.345 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:31:12.345 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 58 M 2024-04-26T04:31:12.345 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-04-26T04:31:12.539 INFO:teuthology.orchestra.run.smithi156.stdout:(5/24): librgw2-17.2.7-904.gd8c6b0a3.el9.x86_64 17 MB/s | 4.9 MB 00:00 2024-04-26T04:31:12.596 INFO:teuthology.orchestra.run.smithi156.stdout:(6/24): python3-ceph-argparse-17.2.7-904.gd8c6b 803 kB/s | 45 kB 00:00 2024-04-26T04:31:12.640 INFO:teuthology.orchestra.run.smithi156.stdout:(7/24): python3-ceph-common-17.2.7-904.gd8c6b0a 2.6 MB/s | 117 kB 00:00 2024-04-26T04:31:12.690 INFO:teuthology.orchestra.run.smithi156.stdout:(8/24): python3-cephfs-17.2.7-904.gd8c6b0a3.el9 3.1 MB/s | 156 kB 00:00 2024-04-26T04:31:12.740 INFO:teuthology.orchestra.run.smithi156.stdout:(9/24): python3-rados-17.2.7-904.gd8c6b0a3.el9. 6.3 MB/s | 320 kB 00:00 2024-04-26T04:31:12.799 INFO:teuthology.orchestra.run.smithi156.stdout:(10/24): python3-rbd-17.2.7-904.gd8c6b0a3.el9.x 4.9 MB/s | 293 kB 00:00 2024-04-26T04:31:12.857 INFO:teuthology.orchestra.run.smithi156.stdout:(11/24): python3-rgw-17.2.7-904.gd8c6b0a3.el9.x 1.7 MB/s | 100 kB 00:00 2024-04-26T04:31:13.249 INFO:teuthology.orchestra.run.smithi156.stdout:(12/24): ceph-base-17.2.7-904.gd8c6b0a3.el9.x86 2.1 MB/s | 5.0 MB 00:02 2024-04-26T04:31:13.908 INFO:teuthology.orchestra.run.smithi156.stdout:(13/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 152 kB/s | 160 kB 00:01 2024-04-26T04:31:13.950 INFO:teuthology.orchestra.run.smithi156.stdout:(14/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 65 kB/s | 45 kB 00:00 2024-04-26T04:31:14.434 INFO:teuthology.orchestra.run.smithi156.stdout:(15/24): librdkafka-1.6.1-102.el9.x86_64.rpm 1.2 MB/s | 662 kB 00:00 2024-04-26T04:31:14.525 INFO:teuthology.orchestra.run.smithi156.stdout:(16/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 508 kB/s | 292 kB 00:00 2024-04-26T04:31:14.626 INFO:teuthology.orchestra.run.smithi156.stdout:(17/24): python3-prettytable-0.7.2-27.el9.noarc 220 kB/s | 42 kB 00:00 2024-04-26T04:31:14.729 INFO:teuthology.orchestra.run.smithi171.stdout:(1/24): ceph-common-17.2.7-904.gd8c6b0a3.el9.x8 13 MB/s | 22 MB 00:01 2024-04-26T04:31:14.779 INFO:teuthology.orchestra.run.smithi171.stdout:(2/24): ceph-selinux-17.2.7-904.gd8c6b0a3.el9.x 507 kB/s | 25 kB 00:00 2024-04-26T04:31:14.876 INFO:teuthology.orchestra.run.smithi156.stdout:(18/24): liboath-2.6.7-2.el9.x86_64.rpm 194 kB/s | 49 kB 00:00 2024-04-26T04:31:14.896 INFO:teuthology.orchestra.run.smithi171.stdout:(3/24): libcephfs2-17.2.7-904.gd8c6b0a3.el9.x86 5.3 MB/s | 637 kB 00:00 2024-04-26T04:31:14.955 INFO:teuthology.orchestra.run.smithi171.stdout:(4/24): libradosstriper1-17.2.7-904.gd8c6b0a3.e 7.7 MB/s | 457 kB 00:00 2024-04-26T04:31:14.968 INFO:teuthology.orchestra.run.smithi156.stdout:(19/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 698 kB/s | 309 kB 00:00 2024-04-26T04:31:15.068 INFO:teuthology.orchestra.run.smithi156.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 351 kB/s | 67 kB 00:00 2024-04-26T04:31:15.280 INFO:teuthology.orchestra.run.smithi171.stdout:(5/24): librgw2-17.2.7-904.gd8c6b0a3.el9.x86_64 15 MB/s | 4.9 MB 00:00 2024-04-26T04:31:15.360 INFO:teuthology.orchestra.run.smithi156.stdout:(21/24): ceph-common-17.2.7-904.gd8c6b0a3.el9.x 5.1 MB/s | 22 MB 00:04 2024-04-26T04:31:15.372 INFO:teuthology.orchestra.run.smithi171.stdout:(6/24): ceph-base-17.2.7-904.gd8c6b0a3.el9.x86_ 2.1 MB/s | 5.0 MB 00:02 2024-04-26T04:31:15.397 INFO:teuthology.orchestra.run.smithi171.stdout:(7/24): python3-ceph-argparse-17.2.7-904.gd8c6b 386 kB/s | 45 kB 00:00 2024-04-26T04:31:15.402 INFO:teuthology.orchestra.run.smithi156.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 3.6 MB/s | 1.6 MB 00:00 2024-04-26T04:31:15.439 INFO:teuthology.orchestra.run.smithi171.stdout:(8/24): python3-ceph-common-17.2.7-904.gd8c6b0a 1.7 MB/s | 117 kB 00:00 2024-04-26T04:31:15.464 INFO:teuthology.orchestra.run.smithi171.stdout:(9/24): python3-cephfs-17.2.7-904.gd8c6b0a3.el9 2.3 MB/s | 156 kB 00:00 2024-04-26T04:31:15.531 INFO:teuthology.orchestra.run.smithi171.stdout:(10/24): python3-rbd-17.2.7-904.gd8c6b0a3.el9.x 4.3 MB/s | 293 kB 00:00 2024-04-26T04:31:15.573 INFO:teuthology.orchestra.run.smithi171.stdout:(11/24): python3-rados-17.2.7-904.gd8c6b0a3.el9 2.3 MB/s | 320 kB 00:00 2024-04-26T04:31:15.599 INFO:teuthology.orchestra.run.smithi171.stdout:(12/24): python3-rgw-17.2.7-904.gd8c6b0a3.el9.x 1.5 MB/s | 100 kB 00:00 2024-04-26T04:31:15.719 INFO:teuthology.orchestra.run.smithi156.stdout:(23/24): librbd1-17.2.7-904.gd8c6b0a3.el9.x86_6 8.2 MB/s | 2.9 MB 00:00 2024-04-26T04:31:15.807 INFO:teuthology.orchestra.run.smithi171.stdout:(13/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 686 kB/s | 160 kB 00:00 2024-04-26T04:31:15.974 INFO:teuthology.orchestra.run.smithi171.stdout:(14/24): librdkafka-1.6.1-102.el9.x86_64.rpm 4.6 MB/s | 662 kB 00:00 2024-04-26T04:31:15.986 INFO:teuthology.orchestra.run.smithi156.stdout:(24/24): librados2-17.2.7-904.gd8c6b0a3.el9.x86 3.4 MB/s | 3.1 MB 00:00 2024-04-26T04:31:15.989 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:15.990 INFO:teuthology.orchestra.run.smithi156.stdout:Total 11 MB/s | 58 MB 00:05 2024-04-26T04:31:16.033 INFO:teuthology.orchestra.run.smithi171.stdout:(15/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 4.9 MB/s | 292 kB 00:00 2024-04-26T04:31:16.053 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-04-26T04:31:16.083 INFO:teuthology.orchestra.run.smithi171.stdout:(16/24): python3-prettytable-0.7.2-27.el9.noarc 847 kB/s | 42 kB 00:00 2024-04-26T04:31:16.088 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-04-26T04:31:16.088 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-04-26T04:31:16.350 INFO:teuthology.orchestra.run.smithi171.stdout:(17/24): ceph-radosgw-17.2.7-904.gd8c6b0a3.el9. 4.2 MB/s | 14 MB 00:03 2024-04-26T04:31:16.375 INFO:teuthology.orchestra.run.smithi171.stdout:(18/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 58 kB/s | 45 kB 00:00 2024-04-26T04:31:16.522 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-04-26T04:31:16.522 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-04-26T04:31:16.725 INFO:teuthology.orchestra.run.smithi171.stdout:(19/24): libunwind-1.6.2-1.el9.x86_64.rpm 192 kB/s | 67 kB 00:00 2024-04-26T04:31:16.759 INFO:teuthology.orchestra.run.smithi171.stdout:(20/24): liboath-2.6.7-2.el9.x86_64.rpm 119 kB/s | 49 kB 00:00 2024-04-26T04:31:16.784 INFO:teuthology.orchestra.run.smithi171.stdout:(21/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 440 kB/s | 309 kB 00:00 2024-04-26T04:31:17.010 INFO:teuthology.orchestra.run.smithi171.stdout:(22/24): librbd1-17.2.7-904.gd8c6b0a3.el9.x86_6 13 MB/s | 2.9 MB 00:00 2024-04-26T04:31:17.634 INFO:teuthology.orchestra.run.smithi171.stdout:(23/24): thrift-0.15.0-2.el9.x86_64.rpm 1.7 MB/s | 1.6 MB 00:00 2024-04-26T04:31:17.726 INFO:teuthology.orchestra.run.smithi171.stdout:(24/24): librados2-17.2.7-904.gd8c6b0a3.el9.x86 3.2 MB/s | 3.1 MB 00:00 2024-04-26T04:31:17.731 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:17.732 INFO:teuthology.orchestra.run.smithi171.stdout:Total 11 MB/s | 58 MB 00:05 2024-04-26T04:31:17.747 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-04-26T04:31:17.796 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-04-26T04:31:17.832 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-04-26T04:31:17.832 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-04-26T04:31:18.152 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-26T04:31:18.213 INFO:teuthology.orchestra.run.smithi156.stdout: Upgrading : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:31:18.267 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-04-26T04:31:18.267 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-04-26T04:31:18.305 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:31:18.383 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:31:18.406 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:31:18.494 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:31:18.540 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-26T04:31:18.570 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-26T04:31:19.005 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-26T04:31:19.026 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:31:19.080 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:31:19.109 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-26T04:31:19.143 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 10/26 2024-04-26T04:31:19.180 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:31:19.242 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:31:19.279 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:31:19.344 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:31:19.383 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-26T04:31:19.412 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-26T04:31:19.445 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-26T04:31:19.472 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-04-26T04:31:19.739 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-26T04:31:19.775 INFO:teuthology.orchestra.run.smithi156.stdout: Upgrading : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:31:19.808 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-26T04:31:19.864 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:31:19.938 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 19/26 2024-04-26T04:31:19.946 INFO:teuthology.orchestra.run.smithi171.stdout: Upgrading : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:31:19.969 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 20/26 2024-04-26T04:31:20.038 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:31:20.116 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:31:20.139 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:31:20.218 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:31:20.264 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-26T04:31:20.293 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-26T04:31:20.731 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-26T04:31:20.758 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:31:20.821 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:31:20.850 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-26T04:31:20.893 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 10/26 2024-04-26T04:31:20.930 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:31:20.992 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:31:21.020 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:31:21.085 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:31:21.124 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-26T04:31:21.153 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-26T04:31:21.186 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-26T04:31:21.479 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-26T04:31:21.532 INFO:teuthology.orchestra.run.smithi171.stdout: Upgrading : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:31:21.613 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:31:21.705 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 19/26 2024-04-26T04:31:21.734 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 20/26 2024-04-26T04:31:22.440 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:31:22.483 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:31:22.993 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:31:23.036 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:31:23.115 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:31:23.115 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-26T04:31:23.115 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-26T04:31:23.115 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:31:23.146 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:31:24.181 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:31:24.215 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:31:24.723 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:31:24.776 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:31:24.875 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:31:24.875 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-26T04:31:24.875 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-26T04:31:24.875 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:31:24.903 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:31:32.561 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:31:32.562 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /sys 2024-04-26T04:31:32.562 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /proc 2024-04-26T04:31:32.562 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /mnt 2024-04-26T04:31:32.562 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /var/tmp 2024-04-26T04:31:32.562 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /home 2024-04-26T04:31:32.562 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /root 2024-04-26T04:31:32.562 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /tmp 2024-04-26T04:31:32.562 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:31:33.863 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:31:33.924 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:31:33.924 INFO:teuthology.orchestra.run.smithi156.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:31:33.925 INFO:teuthology.orchestra.run.smithi156.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-26T04:31:33.925 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:31:33.925 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:31:33.925 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:31:33.926 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:31:34.008 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:31:34.008 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:31:34.333 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:31:34.333 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /sys 2024-04-26T04:31:34.333 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /proc 2024-04-26T04:31:34.333 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /mnt 2024-04-26T04:31:34.334 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /var/tmp 2024-04-26T04:31:34.334 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /home 2024-04-26T04:31:34.334 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /root 2024-04-26T04:31:34.334 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /tmp 2024-04-26T04:31:34.334 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:31:35.287 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:31:35.287 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/26 2024-04-26T04:31:35.287 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:31:35.287 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:31:35.287 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:31:35.287 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/26 2024-04-26T04:31:35.288 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 6/26 2024-04-26T04:31:35.288 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 7/26 2024-04-26T04:31:35.288 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 8/26 2024-04-26T04:31:35.288 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 9/26 2024-04-26T04:31:35.288 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 10/26 2024-04-26T04:31:35.289 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:31:35.289 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:31:35.289 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:31:35.289 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-26T04:31:35.289 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-26T04:31:35.290 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-26T04:31:35.290 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-26T04:31:35.290 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-26T04:31:35.290 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-26T04:31:35.290 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-26T04:31:35.290 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-26T04:31:35.291 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-26T04:31:35.291 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:31:35.291 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-26T04:31:35.291 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 25/26 2024-04-26T04:31:35.612 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:31:35.689 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:31:35.689 INFO:teuthology.orchestra.run.smithi171.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:31:35.690 INFO:teuthology.orchestra.run.smithi171.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-26T04:31:35.690 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:31:35.690 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:31:35.690 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:31:35.691 INFO:teuthology.orchestra.run.smithi171.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:31:35.798 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:31:35.798 INFO:teuthology.orchestra.run.smithi171.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:31:36.025 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:31:36.026 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:31:36.026 INFO:teuthology.orchestra.run.smithi156.stdout:Upgraded: 2024-04-26T04:31:36.026 INFO:teuthology.orchestra.run.smithi156.stdout: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:36.026 INFO:teuthology.orchestra.run.smithi156.stdout: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:36.026 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-04-26T04:31:36.026 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:36.026 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:36.026 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:36.026 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:36.026 INFO:teuthology.orchestra.run.smithi156.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-26T04:31:36.026 INFO:teuthology.orchestra.run.smithi156.stdout: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:36.026 INFO:teuthology.orchestra.run.smithi156.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-26T04:31:36.027 INFO:teuthology.orchestra.run.smithi156.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-26T04:31:36.027 INFO:teuthology.orchestra.run.smithi156.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-26T04:31:36.027 INFO:teuthology.orchestra.run.smithi156.stdout: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:36.027 INFO:teuthology.orchestra.run.smithi156.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-26T04:31:36.027 INFO:teuthology.orchestra.run.smithi156.stdout: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:36.027 INFO:teuthology.orchestra.run.smithi156.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-26T04:31:36.027 INFO:teuthology.orchestra.run.smithi156.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-26T04:31:36.027 INFO:teuthology.orchestra.run.smithi156.stdout: python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:36.027 INFO:teuthology.orchestra.run.smithi156.stdout: python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:36.027 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:36.027 INFO:teuthology.orchestra.run.smithi156.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-26T04:31:36.027 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:36.027 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:36.027 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:36.028 INFO:teuthology.orchestra.run.smithi156.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-26T04:31:36.028 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:31:36.028 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:31:36.410 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-test 2024-04-26T04:31:36.919 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:28 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:31:36.997 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:31:36.997 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/26 2024-04-26T04:31:36.997 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:31:36.997 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:31:36.997 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:31:36.997 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/26 2024-04-26T04:31:36.997 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 6/26 2024-04-26T04:31:36.997 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 7/26 2024-04-26T04:31:36.997 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 8/26 2024-04-26T04:31:36.998 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 9/26 2024-04-26T04:31:36.998 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 10/26 2024-04-26T04:31:36.998 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:31:36.998 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:31:36.998 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:31:36.998 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-26T04:31:36.998 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-26T04:31:36.998 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-26T04:31:36.998 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-26T04:31:36.998 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-26T04:31:36.998 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-26T04:31:36.998 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-26T04:31:36.998 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-26T04:31:36.999 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-26T04:31:36.999 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:31:36.999 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-26T04:31:36.999 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 25/26 2024-04-26T04:31:37.038 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:31:37.038 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:31:37.038 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-04-26T04:31:37.038 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:31:37.038 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-04-26T04:31:37.038 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-test x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 48 M 2024-04-26T04:31:37.039 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2024-04-26T04:31:37.039 INFO:teuthology.orchestra.run.smithi156.stdout: libcephsqlite x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 163 k 2024-04-26T04:31:37.039 INFO:teuthology.orchestra.run.smithi156.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-26T04:31:37.039 INFO:teuthology.orchestra.run.smithi156.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-26T04:31:37.039 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:31:37.039 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:31:37.039 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:31:37.039 INFO:teuthology.orchestra.run.smithi156.stdout:Install 4 Packages 2024-04-26T04:31:37.039 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:31:37.040 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 48 M 2024-04-26T04:31:37.040 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 192 M 2024-04-26T04:31:37.040 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-04-26T04:31:37.422 INFO:teuthology.orchestra.run.smithi156.stdout:(1/4): libcephsqlite-17.2.7-904.gd8c6b0a3.el9.x 644 kB/s | 163 kB 00:00 2024-04-26T04:31:37.610 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:31:37.610 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:31:37.610 INFO:teuthology.orchestra.run.smithi171.stdout:Upgraded: 2024-04-26T04:31:37.610 INFO:teuthology.orchestra.run.smithi171.stdout: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:37.610 INFO:teuthology.orchestra.run.smithi171.stdout: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:37.611 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-04-26T04:31:37.611 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:37.611 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:37.611 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:37.611 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:37.611 INFO:teuthology.orchestra.run.smithi171.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-26T04:31:37.611 INFO:teuthology.orchestra.run.smithi171.stdout: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:37.611 INFO:teuthology.orchestra.run.smithi171.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-26T04:31:37.611 INFO:teuthology.orchestra.run.smithi171.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-26T04:31:37.611 INFO:teuthology.orchestra.run.smithi171.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-26T04:31:37.611 INFO:teuthology.orchestra.run.smithi171.stdout: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:37.611 INFO:teuthology.orchestra.run.smithi171.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-26T04:31:37.611 INFO:teuthology.orchestra.run.smithi171.stdout: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:37.611 INFO:teuthology.orchestra.run.smithi171.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-26T04:31:37.611 INFO:teuthology.orchestra.run.smithi171.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-26T04:31:37.612 INFO:teuthology.orchestra.run.smithi171.stdout: python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:37.612 INFO:teuthology.orchestra.run.smithi171.stdout: python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:37.612 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:37.612 INFO:teuthology.orchestra.run.smithi171.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-26T04:31:37.612 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:37.612 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:37.612 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:37.612 INFO:teuthology.orchestra.run.smithi171.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-26T04:31:37.612 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:31:37.612 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:31:37.930 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-test 2024-04-26T04:31:38.239 INFO:teuthology.orchestra.run.smithi156.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 78 kB/s | 64 kB 00:00 2024-04-26T04:31:38.272 INFO:teuthology.orchestra.run.smithi156.stdout:(3/4): socat-1.7.4.1-5.el9.x86_64.rpm 276 kB/s | 305 kB 00:01 2024-04-26T04:31:38.442 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:28 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:31:38.562 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:31:38.563 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:31:38.563 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-04-26T04:31:38.563 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:31:38.563 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-04-26T04:31:38.563 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-test x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 48 M 2024-04-26T04:31:38.563 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2024-04-26T04:31:38.563 INFO:teuthology.orchestra.run.smithi171.stdout: libcephsqlite x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 163 k 2024-04-26T04:31:38.563 INFO:teuthology.orchestra.run.smithi171.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-26T04:31:38.563 INFO:teuthology.orchestra.run.smithi171.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-26T04:31:38.563 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:31:38.564 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:31:38.564 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:31:38.564 INFO:teuthology.orchestra.run.smithi171.stdout:Install 4 Packages 2024-04-26T04:31:38.564 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:31:38.564 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 48 M 2024-04-26T04:31:38.564 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 192 M 2024-04-26T04:31:38.565 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-04-26T04:31:39.378 INFO:teuthology.orchestra.run.smithi171.stdout:(1/4): libcephsqlite-17.2.7-904.gd8c6b0a3.el9.x 227 kB/s | 163 kB 00:00 2024-04-26T04:31:39.778 INFO:teuthology.orchestra.run.smithi171.stdout:(2/4): socat-1.7.4.1-5.el9.x86_64.rpm 273 kB/s | 305 kB 00:01 2024-04-26T04:31:39.812 INFO:teuthology.orchestra.run.smithi171.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 147 kB/s | 64 kB 00:00 2024-04-26T04:31:39.964 INFO:teuthology.orchestra.run.smithi156.stdout:(4/4): ceph-test-17.2.7-904.gd8c6b0a3.el9.x86_6 17 MB/s | 48 MB 00:02 2024-04-26T04:31:39.970 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:39.970 INFO:teuthology.orchestra.run.smithi156.stdout:Total 17 MB/s | 48 MB 00:02 2024-04-26T04:31:39.997 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-04-26T04:31:40.017 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-04-26T04:31:40.017 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-04-26T04:31:40.380 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-04-26T04:31:40.380 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-04-26T04:31:40.898 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-04-26T04:31:40.982 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-04-26T04:31:41.021 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/4 2024-04-26T04:31:41.041 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:31:42.183 INFO:teuthology.orchestra.run.smithi171.stdout:(4/4): ceph-test-17.2.7-904.gd8c6b0a3.el9.x86_6 14 MB/s | 48 MB 00:03 2024-04-26T04:31:42.186 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:42.187 INFO:teuthology.orchestra.run.smithi171.stdout:Total 13 MB/s | 48 MB 00:03 2024-04-26T04:31:42.214 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-04-26T04:31:42.233 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-04-26T04:31:42.233 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-04-26T04:31:42.594 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-04-26T04:31:42.594 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-04-26T04:31:43.115 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-04-26T04:31:43.188 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-04-26T04:31:43.228 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/4 2024-04-26T04:31:43.255 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:31:45.422 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:31:45.497 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:31:47.571 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:31:47.571 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/4 2024-04-26T04:31:47.571 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/4 2024-04-26T04:31:47.571 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 3/4 2024-04-26T04:31:47.623 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:31:47.734 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:31:47.878 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-04-26T04:31:47.878 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:31:47.878 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-04-26T04:31:47.878 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:47.878 INFO:teuthology.orchestra.run.smithi156.stdout: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:47.878 INFO:teuthology.orchestra.run.smithi156.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-26T04:31:47.878 INFO:teuthology.orchestra.run.smithi156.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-26T04:31:47.878 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:31:47.878 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:31:48.188 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph 2024-04-26T04:31:48.706 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:40 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:31:48.832 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:31:48.835 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:31:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-04-26T04:31:48.835 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:31:48.835 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-04-26T04:31:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: ceph x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 6.5 k 2024-04-26T04:31:48.835 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2024-04-26T04:31:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mds x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.1 M 2024-04-26T04:31:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 1.4 M 2024-04-26T04:31:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-modules-core noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 240 k 2024-04-26T04:31:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mon x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.9 M 2024-04-26T04:31:48.835 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-osd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 15 M 2024-04-26T04:31:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-26T04:31:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-26T04:31:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-26T04:31:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-26T04:31:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-26T04:31:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-26T04:31:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-26T04:31:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-26T04:31:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-26T04:31:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-26T04:31:48.836 INFO:teuthology.orchestra.run.smithi156.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-26T04:31:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-26T04:31:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-26T04:31:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-26T04:31:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-26T04:31:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-26T04:31:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-26T04:31:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-26T04:31:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-26T04:31:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-26T04:31:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-26T04:31:48.837 INFO:teuthology.orchestra.run.smithi156.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-26T04:31:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-26T04:31:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-26T04:31:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-26T04:31:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-26T04:31:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-26T04:31:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:31:48.838 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:31:48.838 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:31:48.838 INFO:teuthology.orchestra.run.smithi156.stdout:Install 33 Packages 2024-04-26T04:31:48.838 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:31:48.839 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 27 M 2024-04-26T04:31:48.839 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 96 M 2024-04-26T04:31:48.839 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-04-26T04:31:49.715 INFO:teuthology.orchestra.run.smithi156.stdout:(1/33): ceph-17.2.7-904.gd8c6b0a3.el9.x86_64.rp 26 kB/s | 6.5 kB 00:00 2024-04-26T04:31:49.849 INFO:teuthology.orchestra.run.smithi156.stdout:(2/33): ceph-mds-17.2.7-904.gd8c6b0a3.el9.x86_6 5.4 MB/s | 2.1 MB 00:00 2024-04-26T04:31:49.895 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:31:49.896 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/4 2024-04-26T04:31:49.896 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/4 2024-04-26T04:31:49.896 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 3/4 2024-04-26T04:31:50.157 INFO:teuthology.orchestra.run.smithi156.stdout:(3/33): ceph-mgr-17.2.7-904.gd8c6b0a3.el9.x86_6 2.0 MB/s | 1.4 MB 00:00 2024-04-26T04:31:50.183 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-04-26T04:31:50.183 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:31:50.183 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-04-26T04:31:50.184 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:50.184 INFO:teuthology.orchestra.run.smithi171.stdout: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:50.184 INFO:teuthology.orchestra.run.smithi171.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-26T04:31:50.184 INFO:teuthology.orchestra.run.smithi171.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-26T04:31:50.184 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:31:50.184 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:31:50.258 INFO:teuthology.orchestra.run.smithi156.stdout:(4/33): ceph-mgr-modules-core-17.2.7-904.gd8c6b 2.3 MB/s | 240 kB 00:00 2024-04-26T04:31:50.451 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph 2024-04-26T04:31:50.625 INFO:teuthology.orchestra.run.smithi156.stdout:(5/33): ceph-osd-17.2.7-904.gd8c6b0a3.el9.x86_6 20 MB/s | 15 MB 00:00 2024-04-26T04:31:50.967 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:40 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:31:51.093 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:31:51.096 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:31:51.096 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-04-26T04:31:51.096 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:31:51.096 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-04-26T04:31:51.096 INFO:teuthology.orchestra.run.smithi171.stdout: ceph x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 6.5 k 2024-04-26T04:31:51.096 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2024-04-26T04:31:51.096 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mds x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.1 M 2024-04-26T04:31:51.096 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 1.4 M 2024-04-26T04:31:51.096 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-modules-core noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 240 k 2024-04-26T04:31:51.096 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mon x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.9 M 2024-04-26T04:31:51.096 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-osd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 15 M 2024-04-26T04:31:51.097 INFO:teuthology.orchestra.run.smithi171.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-26T04:31:51.097 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-26T04:31:51.097 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-26T04:31:51.097 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-26T04:31:51.097 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-26T04:31:51.097 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-26T04:31:51.097 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-26T04:31:51.097 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-26T04:31:51.097 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-26T04:31:51.097 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-26T04:31:51.097 INFO:teuthology.orchestra.run.smithi171.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-26T04:31:51.098 INFO:teuthology.orchestra.run.smithi171.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-26T04:31:51.098 INFO:teuthology.orchestra.run.smithi171.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-26T04:31:51.098 INFO:teuthology.orchestra.run.smithi171.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-26T04:31:51.098 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-26T04:31:51.098 INFO:teuthology.orchestra.run.smithi171.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-26T04:31:51.098 INFO:teuthology.orchestra.run.smithi171.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-26T04:31:51.098 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-26T04:31:51.098 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-26T04:31:51.098 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-26T04:31:51.098 INFO:teuthology.orchestra.run.smithi171.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-26T04:31:51.098 INFO:teuthology.orchestra.run.smithi171.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-26T04:31:51.098 INFO:teuthology.orchestra.run.smithi171.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-26T04:31:51.099 INFO:teuthology.orchestra.run.smithi171.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-26T04:31:51.099 INFO:teuthology.orchestra.run.smithi171.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-26T04:31:51.099 INFO:teuthology.orchestra.run.smithi171.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-26T04:31:51.099 INFO:teuthology.orchestra.run.smithi171.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-26T04:31:51.099 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:31:51.099 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:31:51.099 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:31:51.099 INFO:teuthology.orchestra.run.smithi171.stdout:Install 33 Packages 2024-04-26T04:31:51.099 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:31:51.100 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 27 M 2024-04-26T04:31:51.100 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 96 M 2024-04-26T04:31:51.100 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-04-26T04:31:51.333 INFO:teuthology.orchestra.run.smithi156.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 235 kB/s | 253 kB 00:01 2024-04-26T04:31:51.600 INFO:teuthology.orchestra.run.smithi156.stdout:(7/33): ceph-mon-17.2.7-904.gd8c6b0a3.el9.x86_6 2.1 MB/s | 3.9 MB 00:01 2024-04-26T04:31:51.742 INFO:teuthology.orchestra.run.smithi156.stdout:(8/33): python3-cryptography-36.0.1-4.el9.x86_6 1.1 MB/s | 1.2 MB 00:01 2024-04-26T04:31:51.767 INFO:teuthology.orchestra.run.smithi156.stdout:(9/33): python3-ply-3.11-14.el9.noarch.rpm 246 kB/s | 106 kB 00:00 2024-04-26T04:31:51.792 INFO:teuthology.orchestra.run.smithi156.stdout:(10/33): python3-pycparser-2.20-6.el9.noarch.rp 705 kB/s | 135 kB 00:00 2024-04-26T04:31:51.918 INFO:teuthology.orchestra.run.smithi156.stdout:(11/33): python3-requests-2.25.1-8.el9.noarch.r 711 kB/s | 125 kB 00:00 2024-04-26T04:31:51.968 INFO:teuthology.orchestra.run.smithi156.stdout:(12/33): python3-urllib3-1.26.5-5.el9.noarch.rp 1.0 MB/s | 215 kB 00:00 2024-04-26T04:31:51.994 INFO:teuthology.orchestra.run.smithi156.stdout:(13/33): python3-mako-1.1.4-6.el9.noarch.rpm 857 kB/s | 172 kB 00:00 2024-04-26T04:31:52.094 INFO:teuthology.orchestra.run.smithi156.stdout:(14/33): python3-markupsafe-1.1.1-12.el9.x86_64 198 kB/s | 35 kB 00:00 2024-04-26T04:31:52.152 INFO:teuthology.orchestra.run.smithi156.stdout:(15/33): python3-pytz-2021.1-5.el9.noarch.rpm 278 kB/s | 51 kB 00:00 2024-04-26T04:31:52.178 INFO:teuthology.orchestra.run.smithi156.stdout:(16/33): python3-toml-0.10.2-6.el9.noarch.rpm 226 kB/s | 42 kB 00:00 2024-04-26T04:31:52.179 INFO:teuthology.orchestra.run.smithi171.stdout:(1/33): ceph-17.2.7-904.gd8c6b0a3.el9.x86_64.rp 25 kB/s | 6.5 kB 00:00 2024-04-26T04:31:52.296 INFO:teuthology.orchestra.run.smithi171.stdout:(2/33): ceph-mds-17.2.7-904.gd8c6b0a3.el9.x86_6 5.5 MB/s | 2.1 MB 00:00 2024-04-26T04:31:52.445 INFO:teuthology.orchestra.run.smithi156.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 124 kB/s | 43 kB 00:00 2024-04-26T04:31:52.636 INFO:teuthology.orchestra.run.smithi156.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 56 kB/s | 11 kB 00:00 2024-04-26T04:31:52.670 INFO:teuthology.orchestra.run.smithi156.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 332 kB/s | 172 kB 00:00 2024-04-26T04:31:52.762 INFO:teuthology.orchestra.run.smithi156.stdout:(20/33): python3-cherrypy-18.6.1-2.el9.noarch.r 614 kB/s | 358 kB 00:00 2024-04-26T04:31:52.904 INFO:teuthology.orchestra.run.smithi156.stdout:(21/33): python3-jaraco-classes-3.2.1-5.el9.noa 66 kB/s | 18 kB 00:00 2024-04-26T04:31:52.946 INFO:teuthology.orchestra.run.smithi156.stdout:(22/33): python3-jaraco-collections-3.0.0-8.el9 84 kB/s | 23 kB 00:00 2024-04-26T04:31:53.021 INFO:teuthology.orchestra.run.smithi171.stdout:(3/33): ceph-osd-17.2.7-904.gd8c6b0a3.el9.x86_6 21 MB/s | 15 MB 00:00 2024-04-26T04:31:53.029 INFO:teuthology.orchestra.run.smithi156.stdout:(23/33): python3-jaraco-functools-3.5.0-2.el9.n 73 kB/s | 19 kB 00:00 2024-04-26T04:31:53.075 INFO:teuthology.orchestra.run.smithi171.stdout:(4/33): ceph-mgr-modules-core-17.2.7-904.gd8c6b 4.7 MB/s | 240 kB 00:00 2024-04-26T04:31:53.171 INFO:teuthology.orchestra.run.smithi156.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 74 kB/s | 20 kB 00:00 2024-04-26T04:31:53.230 INFO:teuthology.orchestra.run.smithi171.stdout:(5/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.6 MB/s | 253 kB 00:00 2024-04-26T04:31:53.305 INFO:teuthology.orchestra.run.smithi171.stdout:(6/33): python3-cryptography-36.0.1-4.el9.x86_6 16 MB/s | 1.2 MB 00:00 2024-04-26T04:31:53.339 INFO:teuthology.orchestra.run.smithi171.stdout:(7/33): python3-ply-3.11-14.el9.noarch.rpm 3.1 MB/s | 106 kB 00:00 2024-04-26T04:31:53.355 INFO:teuthology.orchestra.run.smithi156.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 113 kB/s | 46 kB 00:00 2024-04-26T04:31:53.372 INFO:teuthology.orchestra.run.smithi156.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 230 kB/s | 79 kB 00:00 2024-04-26T04:31:53.372 INFO:teuthology.orchestra.run.smithi171.stdout:(8/33): python3-pycparser-2.20-6.el9.noarch.rpm 4.0 MB/s | 135 kB 00:00 2024-04-26T04:31:53.406 INFO:teuthology.orchestra.run.smithi171.stdout:(9/33): python3-requests-2.25.1-8.el9.noarch.rp 3.7 MB/s | 125 kB 00:00 2024-04-26T04:31:53.448 INFO:teuthology.orchestra.run.smithi171.stdout:(10/33): ceph-mgr-17.2.7-904.gd8c6b0a3.el9.x86_ 945 kB/s | 1.4 MB 00:01 2024-04-26T04:31:53.473 INFO:teuthology.orchestra.run.smithi171.stdout:(11/33): python3-urllib3-1.26.5-5.el9.noarch.rp 3.1 MB/s | 215 kB 00:00 2024-04-26T04:31:53.499 INFO:teuthology.orchestra.run.smithi171.stdout:(12/33): python3-mako-1.1.4-6.el9.noarch.rpm 3.4 MB/s | 172 kB 00:00 2024-04-26T04:31:53.524 INFO:teuthology.orchestra.run.smithi171.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 694 kB/s | 35 kB 00:00 2024-04-26T04:31:53.558 INFO:teuthology.orchestra.run.smithi171.stdout:(14/33): python3-pytz-2021.1-5.el9.noarch.rpm 868 kB/s | 51 kB 00:00 2024-04-26T04:31:53.583 INFO:teuthology.orchestra.run.smithi171.stdout:(15/33): python3-toml-0.10.2-6.el9.noarch.rpm 708 kB/s | 42 kB 00:00 2024-04-26T04:31:53.614 INFO:teuthology.orchestra.run.smithi156.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 63 kB/s | 16 kB 00:00 2024-04-26T04:31:53.681 INFO:teuthology.orchestra.run.smithi156.stdout:(28/33): python3-pecan-1.4.2-3.el9.noarch.rpm 535 kB/s | 272 kB 00:00 2024-04-26T04:31:53.789 INFO:teuthology.orchestra.run.smithi156.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 215 kB/s | 90 kB 00:00 2024-04-26T04:31:53.833 INFO:teuthology.orchestra.run.smithi171.stdout:(16/33): ceph-mon-17.2.7-904.gd8c6b0a3.el9.x86_ 2.4 MB/s | 3.9 MB 00:01 2024-04-26T04:31:53.873 INFO:teuthology.orchestra.run.smithi156.stdout:(30/33): python3-tempora-5.0.0-2.el9.noarch.rpm 138 kB/s | 36 kB 00:00 2024-04-26T04:31:53.894 INFO:teuthology.orchestra.run.smithi171.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 130 kB/s | 43 kB 00:00 2024-04-26T04:31:53.992 INFO:teuthology.orchestra.run.smithi171.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 107 kB/s | 11 kB 00:00 2024-04-26T04:31:54.042 INFO:teuthology.orchestra.run.smithi171.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 374 kB/s | 172 kB 00:00 2024-04-26T04:31:54.101 INFO:teuthology.orchestra.run.smithi171.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 163 kB/s | 18 kB 00:00 2024-04-26T04:31:54.140 INFO:teuthology.orchestra.run.smithi156.stdout:(31/33): python3-zc-lockfile-2.0-10.el9.noarch. 75 kB/s | 20 kB 00:00 2024-04-26T04:31:54.143 INFO:teuthology.orchestra.run.smithi171.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 231 kB/s | 23 kB 00:00 2024-04-26T04:31:54.190 INFO:teuthology.orchestra.run.smithi156.stdout:(32/33): python3-webob-1.8.7-6.el9.noarch.rpm 451 kB/s | 230 kB 00:00 2024-04-26T04:31:54.210 INFO:teuthology.orchestra.run.smithi171.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 179 kB/s | 19 kB 00:00 2024-04-26T04:31:54.243 INFO:teuthology.orchestra.run.smithi171.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 196 kB/s | 20 kB 00:00 2024-04-26T04:31:54.352 INFO:teuthology.orchestra.run.smithi171.stdout:(24/33): python3-logutils-0.3.5-21.el9.noarch.r 326 kB/s | 46 kB 00:00 2024-04-26T04:31:54.378 INFO:teuthology.orchestra.run.smithi171.stdout:(25/33): python3-cherrypy-18.6.1-2.el9.noarch.r 659 kB/s | 358 kB 00:00 2024-04-26T04:31:54.403 INFO:teuthology.orchestra.run.smithi171.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 496 kB/s | 79 kB 00:00 2024-04-26T04:31:54.478 INFO:teuthology.orchestra.run.smithi171.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 164 kB/s | 16 kB 00:00 2024-04-26T04:31:54.511 INFO:teuthology.orchestra.run.smithi171.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 828 kB/s | 90 kB 00:00 2024-04-26T04:31:54.587 INFO:teuthology.orchestra.run.smithi171.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 329 kB/s | 36 kB 00:00 2024-04-26T04:31:54.615 INFO:teuthology.orchestra.run.smithi156.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 518 kB/s | 427 kB 00:00 2024-04-26T04:31:54.620 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:54.620 INFO:teuthology.orchestra.run.smithi156.stdout:Total 4.7 MB/s | 27 MB 00:05 2024-04-26T04:31:54.637 INFO:teuthology.orchestra.run.smithi171.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 957 kB/s | 272 kB 00:00 2024-04-26T04:31:54.662 INFO:teuthology.orchestra.run.smithi171.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 1.5 MB/s | 230 kB 00:00 2024-04-26T04:31:54.729 INFO:teuthology.orchestra.run.smithi171.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 217 kB/s | 20 kB 00:00 2024-04-26T04:31:54.738 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-04-26T04:31:54.776 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-04-26T04:31:54.776 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-04-26T04:31:54.813 INFO:teuthology.orchestra.run.smithi171.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 1.9 MB/s | 427 kB 00:00 2024-04-26T04:31:54.817 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:54.817 INFO:teuthology.orchestra.run.smithi171.stdout:Total 7.3 MB/s | 27 MB 00:03 2024-04-26T04:31:54.934 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-04-26T04:31:54.972 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-04-26T04:31:54.972 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-04-26T04:31:55.047 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-04-26T04:31:55.047 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-04-26T04:31:55.237 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-04-26T04:31:55.237 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-04-26T04:31:56.181 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-04-26T04:31:56.251 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-26T04:31:56.283 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-26T04:31:56.316 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-26T04:31:56.409 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-26T04:31:56.412 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-04-26T04:31:56.466 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-26T04:31:56.480 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-26T04:31:56.498 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-26T04:31:56.531 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-26T04:31:56.536 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-26T04:31:56.571 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-26T04:31:56.601 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-04-26T04:31:56.624 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-26T04:31:56.635 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-04-26T04:31:56.668 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-04-26T04:31:56.703 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-26T04:31:56.709 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-04-26T04:31:56.744 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-04-26T04:31:56.750 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-26T04:31:56.779 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-04-26T04:31:56.786 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-26T04:31:56.815 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-04-26T04:31:56.817 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-04-26T04:31:56.850 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-04-26T04:31:56.883 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-04-26T04:31:56.890 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-04-26T04:31:56.924 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-04-26T04:31:56.963 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-04-26T04:31:56.993 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-04-26T04:31:57.031 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-04-26T04:31:57.071 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-04-26T04:31:57.114 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-04-26T04:31:57.166 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-04-26T04:31:57.220 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 18/33 2024-04-26T04:31:57.254 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-04-26T04:31:57.258 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-04-26T04:31:57.315 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-04-26T04:31:57.340 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-04-26T04:31:57.373 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-04-26T04:31:57.393 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 18/33 2024-04-26T04:31:57.431 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-04-26T04:31:57.488 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-04-26T04:31:57.552 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-04-26T04:31:57.557 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-04-26T04:31:57.604 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-04-26T04:31:57.681 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-04-26T04:31:57.742 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-04-26T04:31:57.770 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-04-26T04:31:57.802 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-04-26T04:31:57.811 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-04-26T04:31:57.881 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-04-26T04:31:57.926 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-04-26T04:31:57.994 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-04-26T04:31:58.034 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-04-26T04:31:58.083 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 28/33 2024-04-26T04:31:58.113 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:31:58.138 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:31:58.138 INFO:teuthology.orchestra.run.smithi156.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:31:58.138 INFO:teuthology.orchestra.run.smithi156.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-26T04:31:58.138 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:31:58.138 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:31:58.138 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:31:58.151 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-04-26T04:31:58.305 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 28/33 2024-04-26T04:31:58.336 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:31:58.361 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:31:58.361 INFO:teuthology.orchestra.run.smithi171.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:31:58.361 INFO:teuthology.orchestra.run.smithi171.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-26T04:31:58.361 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:31:58.361 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:31:58.361 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:31:59.498 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:31:59.526 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:31:59.526 INFO:teuthology.orchestra.run.smithi156.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:31:59.526 INFO:teuthology.orchestra.run.smithi156.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-26T04:31:59.526 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:31:59.526 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:31:59.526 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:31:59.746 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:31:59.773 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:31:59.773 INFO:teuthology.orchestra.run.smithi171.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:31:59.774 INFO:teuthology.orchestra.run.smithi171.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-26T04:31:59.774 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:31:59.774 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:31:59.774 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:31:59.907 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:31:59.932 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:31:59.932 INFO:teuthology.orchestra.run.smithi156.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:31:59.932 INFO:teuthology.orchestra.run.smithi156.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-26T04:31:59.932 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:31:59.932 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:31:59.932 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:00.133 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:32:00.155 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:32:00.156 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:32:00.156 INFO:teuthology.orchestra.run.smithi156.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:32:00.156 INFO:teuthology.orchestra.run.smithi156.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-26T04:32:00.156 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:32:00.156 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:32:00.156 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:00.178 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:32:00.179 INFO:teuthology.orchestra.run.smithi171.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:32:00.179 INFO:teuthology.orchestra.run.smithi171.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-26T04:32:00.179 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:32:00.179 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:32:00.179 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:00.220 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:32:00.380 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:32:00.404 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:32:00.404 INFO:teuthology.orchestra.run.smithi171.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:32:00.404 INFO:teuthology.orchestra.run.smithi171.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-26T04:32:00.404 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:32:00.404 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:32:00.404 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:00.468 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:32:01.810 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:32:01.810 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/33 2024-04-26T04:32:01.810 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/33 2024-04-26T04:32:01.810 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/33 2024-04-26T04:32:01.810 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/33 2024-04-26T04:32:01.810 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/33 2024-04-26T04:32:01.810 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 6/33 2024-04-26T04:32:01.811 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-04-26T04:32:01.811 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-04-26T04:32:01.811 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-04-26T04:32:01.811 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-04-26T04:32:01.811 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-04-26T04:32:01.811 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 12/33 2024-04-26T04:32:01.811 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-04-26T04:32:01.811 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-04-26T04:32:01.811 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-04-26T04:32:01.811 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-26T04:32:01.811 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-26T04:32:01.811 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-26T04:32:01.811 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-26T04:32:01.811 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-26T04:32:01.811 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-26T04:32:01.812 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-26T04:32:01.812 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-26T04:32:01.812 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-26T04:32:01.812 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-26T04:32:01.812 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-26T04:32:01.812 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-26T04:32:01.812 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-26T04:32:01.812 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-26T04:32:01.812 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-26T04:32:01.812 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-26T04:32:01.812 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-26T04:32:02.075 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:32:02.076 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/33 2024-04-26T04:32:02.076 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/33 2024-04-26T04:32:02.076 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/33 2024-04-26T04:32:02.076 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/33 2024-04-26T04:32:02.076 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/33 2024-04-26T04:32:02.076 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 6/33 2024-04-26T04:32:02.076 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-04-26T04:32:02.076 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-04-26T04:32:02.076 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-04-26T04:32:02.076 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-04-26T04:32:02.076 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-04-26T04:32:02.077 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 12/33 2024-04-26T04:32:02.077 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-04-26T04:32:02.077 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-04-26T04:32:02.077 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-04-26T04:32:02.077 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-26T04:32:02.077 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-26T04:32:02.077 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-26T04:32:02.077 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-26T04:32:02.077 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-26T04:32:02.078 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-26T04:32:02.078 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-26T04:32:02.078 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-26T04:32:02.078 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-26T04:32:02.078 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-26T04:32:02.079 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-26T04:32:02.079 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-26T04:32:02.079 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-26T04:32:02.079 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-26T04:32:02.079 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-26T04:32:02.079 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-26T04:32:02.079 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-26T04:32:02.640 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-26T04:32:02.641 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:02.641 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-04-26T04:32:02.641 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:02.641 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:02.641 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:02.641 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:32:02.641 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:02.641 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:02.641 INFO:teuthology.orchestra.run.smithi156.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-26T04:32:02.641 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-26T04:32:02.641 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-26T04:32:02.641 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-26T04:32:02.641 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-26T04:32:02.642 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-26T04:32:02.642 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-26T04:32:02.642 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-26T04:32:02.642 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-26T04:32:02.642 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-26T04:32:02.642 INFO:teuthology.orchestra.run.smithi156.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-26T04:32:02.642 INFO:teuthology.orchestra.run.smithi156.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-26T04:32:02.642 INFO:teuthology.orchestra.run.smithi156.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-26T04:32:02.642 INFO:teuthology.orchestra.run.smithi156.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-26T04:32:02.642 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-26T04:32:02.642 INFO:teuthology.orchestra.run.smithi156.stdout: python3-ply-3.11-14.el9.noarch 2024-04-26T04:32:02.642 INFO:teuthology.orchestra.run.smithi156.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-26T04:32:02.642 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-26T04:32:02.642 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-26T04:32:02.643 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-26T04:32:02.643 INFO:teuthology.orchestra.run.smithi156.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-26T04:32:02.643 INFO:teuthology.orchestra.run.smithi156.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-26T04:32:02.643 INFO:teuthology.orchestra.run.smithi156.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-26T04:32:02.643 INFO:teuthology.orchestra.run.smithi156.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-26T04:32:02.643 INFO:teuthology.orchestra.run.smithi156.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-26T04:32:02.643 INFO:teuthology.orchestra.run.smithi156.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-26T04:32:02.643 INFO:teuthology.orchestra.run.smithi156.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-26T04:32:02.643 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:02.643 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:03.002 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-base 2024-04-26T04:32:03.064 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-26T04:32:03.064 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:03.064 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-04-26T04:32:03.064 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:03.064 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:03.064 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:03.064 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:32:03.064 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:03.064 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:03.064 INFO:teuthology.orchestra.run.smithi171.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-26T04:32:03.064 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-26T04:32:03.065 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-26T04:32:03.065 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-26T04:32:03.065 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-26T04:32:03.065 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-26T04:32:03.065 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-26T04:32:03.065 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-26T04:32:03.065 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-26T04:32:03.065 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-26T04:32:03.065 INFO:teuthology.orchestra.run.smithi171.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-26T04:32:03.065 INFO:teuthology.orchestra.run.smithi171.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-26T04:32:03.065 INFO:teuthology.orchestra.run.smithi171.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-26T04:32:03.065 INFO:teuthology.orchestra.run.smithi171.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-26T04:32:03.065 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-26T04:32:03.065 INFO:teuthology.orchestra.run.smithi171.stdout: python3-ply-3.11-14.el9.noarch 2024-04-26T04:32:03.065 INFO:teuthology.orchestra.run.smithi171.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-26T04:32:03.066 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-26T04:32:03.066 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-26T04:32:03.066 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-26T04:32:03.066 INFO:teuthology.orchestra.run.smithi171.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-26T04:32:03.066 INFO:teuthology.orchestra.run.smithi171.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-26T04:32:03.066 INFO:teuthology.orchestra.run.smithi171.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-26T04:32:03.066 INFO:teuthology.orchestra.run.smithi171.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-26T04:32:03.066 INFO:teuthology.orchestra.run.smithi171.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-26T04:32:03.066 INFO:teuthology.orchestra.run.smithi171.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-26T04:32:03.066 INFO:teuthology.orchestra.run.smithi171.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-26T04:32:03.066 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:03.066 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:03.375 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-base 2024-04-26T04:32:03.520 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:55 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:03.599 INFO:teuthology.orchestra.run.smithi156.stdout:Package ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:03.639 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:03.640 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-04-26T04:32:03.640 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:03.723 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install cephadm 2024-04-26T04:32:03.897 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:53 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:03.977 INFO:teuthology.orchestra.run.smithi171.stdout:Package ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:04.018 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:04.019 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-04-26T04:32:04.019 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:04.082 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install cephadm 2024-04-26T04:32:04.236 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:56 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:04.356 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:04.357 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:04.357 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-04-26T04:32:04.357 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:04.357 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-04-26T04:32:04.357 INFO:teuthology.orchestra.run.smithi156.stdout: cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 82 k 2024-04-26T04:32:04.357 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:04.357 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:32:04.357 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:04.357 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2024-04-26T04:32:04.358 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:04.358 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 82 k 2024-04-26T04:32:04.358 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 372 k 2024-04-26T04:32:04.358 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-04-26T04:32:04.566 INFO:teuthology.orchestra.run.smithi156.stdout:cephadm-17.2.7-904.gd8c6b0a3.el9.noarch.rpm 395 kB/s | 82 kB 00:00 2024-04-26T04:32:04.566 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:04.566 INFO:teuthology.orchestra.run.smithi156.stdout:Total 392 kB/s | 82 kB 00:00 2024-04-26T04:32:04.567 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-04-26T04:32:04.572 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-04-26T04:32:04.572 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-04-26T04:32:04.574 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-04-26T04:32:04.574 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-04-26T04:32:04.596 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:54 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:04.639 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-04-26T04:32:04.717 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:04.717 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:04.717 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-04-26T04:32:04.718 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:04.718 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-04-26T04:32:04.718 INFO:teuthology.orchestra.run.smithi171.stdout: cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 82 k 2024-04-26T04:32:04.718 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:04.718 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:32:04.718 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:04.718 INFO:teuthology.orchestra.run.smithi171.stdout:Install 1 Package 2024-04-26T04:32:04.718 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:04.718 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 82 k 2024-04-26T04:32:04.718 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 372 k 2024-04-26T04:32:04.718 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-04-26T04:32:05.063 INFO:teuthology.orchestra.run.smithi171.stdout:cephadm-17.2.7-904.gd8c6b0a3.el9.noarch.rpm 238 kB/s | 82 kB 00:00 2024-04-26T04:32:05.064 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:05.064 INFO:teuthology.orchestra.run.smithi171.stdout:Total 237 kB/s | 82 kB 00:00 2024-04-26T04:32:05.064 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-04-26T04:32:05.070 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-04-26T04:32:05.070 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-04-26T04:32:05.071 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-04-26T04:32:05.072 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-04-26T04:32:05.081 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:32:05.150 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-04-26T04:32:05.185 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:32:05.514 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:32:05.612 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:32:05.716 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:32:05.732 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:32:05.732 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:05.732 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-04-26T04:32:05.732 INFO:teuthology.orchestra.run.smithi156.stdout: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:32:05.732 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:05.732 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:05.955 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-immutable-object-cache 2024-04-26T04:32:06.027 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:32:06.246 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:32:06.246 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:06.246 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-04-26T04:32:06.246 INFO:teuthology.orchestra.run.smithi171.stdout: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:32:06.246 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:06.247 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:06.434 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-immutable-object-cache 2024-04-26T04:32:06.474 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:58 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:06.597 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:06.597 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:06.597 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repo Size 2024-04-26T04:32:06.598 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:06.598 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-04-26T04:32:06.598 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 137 k 2024-04-26T04:32:06.598 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:06.598 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:32:06.598 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:06.598 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2024-04-26T04:32:06.598 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:06.598 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 137 k 2024-04-26T04:32:06.598 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 379 k 2024-04-26T04:32:06.598 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-04-26T04:32:06.952 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:56 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:07.014 INFO:teuthology.orchestra.run.smithi156.stdout:ceph-immutable-object-cache-17.2.7-904.gd8c6b0a 330 kB/s | 137 kB 00:00 2024-04-26T04:32:07.015 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:07.015 INFO:teuthology.orchestra.run.smithi156.stdout:Total 329 kB/s | 137 kB 00:00 2024-04-26T04:32:07.015 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-04-26T04:32:07.023 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-04-26T04:32:07.023 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-04-26T04:32:07.075 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:07.076 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:07.076 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repo Size 2024-04-26T04:32:07.076 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:07.076 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-04-26T04:32:07.076 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 137 k 2024-04-26T04:32:07.076 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:07.077 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:32:07.077 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:07.077 INFO:teuthology.orchestra.run.smithi171.stdout:Install 1 Package 2024-04-26T04:32:07.077 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:07.077 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 137 k 2024-04-26T04:32:07.077 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 379 k 2024-04-26T04:32:07.077 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-04-26T04:32:07.079 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-04-26T04:32:07.080 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-04-26T04:32:07.234 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-04-26T04:32:07.332 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:32:07.357 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:32:07.357 INFO:teuthology.orchestra.run.smithi156.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:32:07.357 INFO:teuthology.orchestra.run.smithi156.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-26T04:32:07.357 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:07.553 INFO:teuthology.orchestra.run.smithi171.stdout:ceph-immutable-object-cache-17.2.7-904.gd8c6b0a 289 kB/s | 137 kB 00:00 2024-04-26T04:32:07.554 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:07.554 INFO:teuthology.orchestra.run.smithi171.stdout:Total 287 kB/s | 137 kB 00:00 2024-04-26T04:32:07.554 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-04-26T04:32:07.562 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-04-26T04:32:07.562 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-04-26T04:32:07.619 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-04-26T04:32:07.619 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-04-26T04:32:07.767 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-04-26T04:32:07.846 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:32:07.870 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:32:07.870 INFO:teuthology.orchestra.run.smithi171.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:32:07.870 INFO:teuthology.orchestra.run.smithi171.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-26T04:32:07.870 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:08.196 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:32:08.196 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:08.196 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-04-26T04:32:08.196 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:08.197 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:08.197 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:08.401 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-mgr 2024-04-26T04:32:08.719 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:32:08.719 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:08.719 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-04-26T04:32:08.719 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:08.719 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:08.719 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:08.920 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:00 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:08.947 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-mgr 2024-04-26T04:32:09.001 INFO:teuthology.orchestra.run.smithi156.stdout:Package ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:09.044 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:09.045 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-04-26T04:32:09.045 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:09.116 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-mgr-dashboard 2024-04-26T04:32:09.472 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:59 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:09.552 INFO:teuthology.orchestra.run.smithi171.stdout:Package ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:09.593 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:09.594 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-04-26T04:32:09.594 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:09.630 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:01 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:09.678 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-mgr-dashboard 2024-04-26T04:32:09.751 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:09.751 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:09.751 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-04-26T04:32:09.752 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:09.752 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-04-26T04:32:09.752 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-dashboard noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 3.4 M 2024-04-26T04:32:09.752 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2024-04-26T04:32:09.752 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-grafana-dashboards noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 24 k 2024-04-26T04:32:09.752 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-prometheus-alerts noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 15 k 2024-04-26T04:32:09.752 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-26T04:32:09.752 INFO:teuthology.orchestra.run.smithi156.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-26T04:32:09.752 INFO:teuthology.orchestra.run.smithi156.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-26T04:32:09.752 INFO:teuthology.orchestra.run.smithi156.stdout:Installing weak dependencies: 2024-04-26T04:32:09.752 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-26T04:32:09.752 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:09.752 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:32:09.752 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:09.753 INFO:teuthology.orchestra.run.smithi156.stdout:Install 7 Packages 2024-04-26T04:32:09.753 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:09.753 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 3.7 M 2024-04-26T04:32:09.753 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 80 M 2024-04-26T04:32:09.753 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-04-26T04:32:10.207 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:00 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:10.309 INFO:teuthology.orchestra.run.smithi156.stdout:(1/7): ceph-grafana-dashboards-17.2.7-904.gd8c6 133 kB/s | 24 kB 00:00 2024-04-26T04:32:10.329 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:10.330 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-04-26T04:32:10.330 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:10.330 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-04-26T04:32:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-dashboard noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 3.4 M 2024-04-26T04:32:10.330 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2024-04-26T04:32:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-grafana-dashboards noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 24 k 2024-04-26T04:32:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-prometheus-alerts noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 15 k 2024-04-26T04:32:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-26T04:32:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-26T04:32:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-26T04:32:10.331 INFO:teuthology.orchestra.run.smithi171.stdout:Installing weak dependencies: 2024-04-26T04:32:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-26T04:32:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:10.331 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:32:10.331 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:10.331 INFO:teuthology.orchestra.run.smithi171.stdout:Install 7 Packages 2024-04-26T04:32:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:10.332 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 3.7 M 2024-04-26T04:32:10.332 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 80 M 2024-04-26T04:32:10.332 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-04-26T04:32:10.418 INFO:teuthology.orchestra.run.smithi156.stdout:(2/7): ceph-prometheus-alerts-17.2.7-904.gd8c6b 50 kB/s | 15 kB 00:00 2024-04-26T04:32:10.577 INFO:teuthology.orchestra.run.smithi156.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 34 kB/s | 9.0 kB 00:00 2024-04-26T04:32:10.610 INFO:teuthology.orchestra.run.smithi156.stdout:(4/7): python3-jwt-2.4.0-1.el9.noarch.rpm 243 kB/s | 41 kB 00:00 2024-04-26T04:32:10.635 INFO:teuthology.orchestra.run.smithi156.stdout:(5/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 525 kB/s | 31 kB 00:00 2024-04-26T04:32:10.666 INFO:teuthology.orchestra.run.smithi171.stdout:(1/7): ceph-prometheus-alerts-17.2.7-904.gd8c6b 59 kB/s | 15 kB 00:00 2024-04-26T04:32:10.691 INFO:teuthology.orchestra.run.smithi171.stdout:(2/7): ceph-grafana-dashboards-17.2.7-904.gd8c6 89 kB/s | 24 kB 00:00 2024-04-26T04:32:10.719 INFO:teuthology.orchestra.run.smithi156.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 1.7 MB/s | 188 kB 00:00 2024-04-26T04:32:10.927 INFO:teuthology.orchestra.run.smithi156.stdout:(7/7): ceph-mgr-dashboard-17.2.7-904.gd8c6b0a3. 4.2 MB/s | 3.4 MB 00:00 2024-04-26T04:32:10.931 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:10.931 INFO:teuthology.orchestra.run.smithi156.stdout:Total 3.1 MB/s | 3.7 MB 00:01 2024-04-26T04:32:10.931 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-04-26T04:32:10.940 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-04-26T04:32:10.940 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-04-26T04:32:11.001 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-04-26T04:32:11.001 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-04-26T04:32:11.091 INFO:teuthology.orchestra.run.smithi171.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 21 kB/s | 9.0 kB 00:00 2024-04-26T04:32:11.166 INFO:teuthology.orchestra.run.smithi171.stdout:(4/7): ceph-mgr-dashboard-17.2.7-904.gd8c6b0a3. 4.5 MB/s | 3.4 MB 00:00 2024-04-26T04:32:11.266 INFO:teuthology.orchestra.run.smithi171.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 71 kB/s | 41 kB 00:00 2024-04-26T04:32:11.307 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-04-26T04:32:11.316 INFO:teuthology.orchestra.run.smithi171.stdout:(6/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 136 kB/s | 31 kB 00:00 2024-04-26T04:32:11.371 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-26T04:32:11.404 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-26T04:32:11.451 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-26T04:32:11.483 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-26T04:32:11.512 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 5/7 2024-04-26T04:32:12.000 INFO:teuthology.orchestra.run.smithi171.stdout:(7/7): python3-routes-2.5.1-5.el9.noarch.rpm 226 kB/s | 188 kB 00:00 2024-04-26T04:32:12.005 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:12.005 INFO:teuthology.orchestra.run.smithi171.stdout:Total 2.2 MB/s | 3.7 MB 00:01 2024-04-26T04:32:12.006 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-04-26T04:32:12.014 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-04-26T04:32:12.015 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-04-26T04:32:12.079 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-04-26T04:32:12.081 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-04-26T04:32:12.391 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-04-26T04:32:12.468 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-26T04:32:12.502 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-26T04:32:12.515 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 6/7 2024-04-26T04:32:12.548 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-26T04:32:12.557 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:32:12.580 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-26T04:32:12.609 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 5/7 2024-04-26T04:32:13.614 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 6/7 2024-04-26T04:32:13.640 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:32:13.640 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 1/7 2024-04-26T04:32:13.640 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2/7 2024-04-26T04:32:13.640 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 3/7 2024-04-26T04:32:13.640 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-26T04:32:13.640 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-26T04:32:13.640 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-26T04:32:13.646 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:32:13.884 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-26T04:32:13.884 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:13.884 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-04-26T04:32:13.884 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:32:13.885 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:32:13.885 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:32:13.885 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-26T04:32:13.885 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-26T04:32:13.885 INFO:teuthology.orchestra.run.smithi156.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-26T04:32:13.885 INFO:teuthology.orchestra.run.smithi156.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-26T04:32:13.886 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:13.886 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:14.093 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-26T04:32:14.605 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:06 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:14.726 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:14.727 INFO:teuthology.orchestra.run.smithi156.stdout:====================================================================================== 2024-04-26T04:32:14.727 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-04-26T04:32:14.727 INFO:teuthology.orchestra.run.smithi156.stdout:====================================================================================== 2024-04-26T04:32:14.727 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-04-26T04:32:14.727 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 7.4 M 2024-04-26T04:32:14.727 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2024-04-26T04:32:14.727 INFO:teuthology.orchestra.run.smithi156.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-26T04:32:14.727 INFO:teuthology.orchestra.run.smithi156.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-26T04:32:14.728 INFO:teuthology.orchestra.run.smithi156.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-26T04:32:14.728 INFO:teuthology.orchestra.run.smithi156.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-26T04:32:14.728 INFO:teuthology.orchestra.run.smithi156.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-26T04:32:14.728 INFO:teuthology.orchestra.run.smithi156.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-26T04:32:14.728 INFO:teuthology.orchestra.run.smithi156.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-26T04:32:14.728 INFO:teuthology.orchestra.run.smithi156.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-04-26T04:32:14.728 INFO:teuthology.orchestra.run.smithi156.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-26T04:32:14.728 INFO:teuthology.orchestra.run.smithi156.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-26T04:32:14.728 INFO:teuthology.orchestra.run.smithi156.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-26T04:32:14.728 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:14.728 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:32:14.728 INFO:teuthology.orchestra.run.smithi156.stdout:====================================================================================== 2024-04-26T04:32:14.728 INFO:teuthology.orchestra.run.smithi156.stdout:Install 12 Packages 2024-04-26T04:32:14.728 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:14.729 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 38 M 2024-04-26T04:32:14.729 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 209 M 2024-04-26T04:32:14.729 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-04-26T04:32:14.739 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:32:14.739 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 1/7 2024-04-26T04:32:14.739 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2/7 2024-04-26T04:32:14.739 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 3/7 2024-04-26T04:32:14.739 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-26T04:32:14.739 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-26T04:32:14.739 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-26T04:32:14.995 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-26T04:32:14.995 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:14.995 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-04-26T04:32:14.996 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:32:14.996 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:32:14.996 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:32:14.996 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-26T04:32:14.996 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-26T04:32:14.996 INFO:teuthology.orchestra.run.smithi171.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-26T04:32:14.996 INFO:teuthology.orchestra.run.smithi171.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-26T04:32:14.996 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:14.996 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:15.202 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-26T04:32:15.494 INFO:teuthology.orchestra.run.smithi156.stdout:(1/12): libquadmath-11.4.1-3.el9.x86_64.rpm 772 kB/s | 195 kB 00:00 2024-04-26T04:32:15.578 INFO:teuthology.orchestra.run.smithi156.stdout:(2/12): libgfortran-11.4.1-3.el9.x86_64.rpm 2.3 MB/s | 803 kB 00:00 2024-04-26T04:32:15.603 INFO:teuthology.orchestra.run.smithi156.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 301 kB/s | 33 kB 00:00 2024-04-26T04:32:15.720 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:05 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:15.736 INFO:teuthology.orchestra.run.smithi156.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 136 kB/s | 18 kB 00:00 2024-04-26T04:32:15.837 INFO:teuthology.orchestra.run.smithi156.stdout:(5/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 12 MB/s | 3.0 MB 00:00 2024-04-26T04:32:15.842 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:15.843 INFO:teuthology.orchestra.run.smithi171.stdout:====================================================================================== 2024-04-26T04:32:15.843 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-04-26T04:32:15.843 INFO:teuthology.orchestra.run.smithi171.stdout:====================================================================================== 2024-04-26T04:32:15.843 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-04-26T04:32:15.843 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 7.4 M 2024-04-26T04:32:15.844 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2024-04-26T04:32:15.844 INFO:teuthology.orchestra.run.smithi171.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-26T04:32:15.844 INFO:teuthology.orchestra.run.smithi171.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-26T04:32:15.844 INFO:teuthology.orchestra.run.smithi171.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-26T04:32:15.844 INFO:teuthology.orchestra.run.smithi171.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-26T04:32:15.844 INFO:teuthology.orchestra.run.smithi171.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-26T04:32:15.844 INFO:teuthology.orchestra.run.smithi171.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-26T04:32:15.844 INFO:teuthology.orchestra.run.smithi171.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-26T04:32:15.844 INFO:teuthology.orchestra.run.smithi171.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-04-26T04:32:15.844 INFO:teuthology.orchestra.run.smithi171.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-26T04:32:15.844 INFO:teuthology.orchestra.run.smithi171.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-26T04:32:15.844 INFO:teuthology.orchestra.run.smithi171.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-26T04:32:15.844 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:15.844 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:32:15.845 INFO:teuthology.orchestra.run.smithi171.stdout:====================================================================================== 2024-04-26T04:32:15.845 INFO:teuthology.orchestra.run.smithi171.stdout:Install 12 Packages 2024-04-26T04:32:15.845 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:15.845 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 38 M 2024-04-26T04:32:15.845 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 209 M 2024-04-26T04:32:15.846 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-04-26T04:32:15.862 INFO:teuthology.orchestra.run.smithi156.stdout:(6/12): openblas-0.3.21-2.el9.x86_64.rpm 283 kB/s | 35 kB 00:00 2024-04-26T04:32:15.987 INFO:teuthology.orchestra.run.smithi156.stdout:(7/12): python3-devel-3.9.18-3.el9.x86_64.rpm 1.9 MB/s | 244 kB 00:00 2024-04-26T04:32:16.063 INFO:teuthology.orchestra.run.smithi156.stdout:(8/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 21 MB/s | 4.7 MB 00:00 2024-04-26T04:32:16.230 INFO:teuthology.orchestra.run.smithi156.stdout:(9/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 21 MB/s | 5.1 MB 00:00 2024-04-26T04:32:16.255 INFO:teuthology.orchestra.run.smithi156.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 1.3 MB/s | 250 kB 00:00 2024-04-26T04:32:16.583 INFO:teuthology.orchestra.run.smithi171.stdout:(1/12): libquadmath-11.4.1-3.el9.x86_64.rpm 503 kB/s | 195 kB 00:00 2024-04-26T04:32:16.725 INFO:teuthology.orchestra.run.smithi171.stdout:(2/12): libgfortran-11.4.1-3.el9.x86_64.rpm 1.5 MB/s | 803 kB 00:00 2024-04-26T04:32:16.750 INFO:teuthology.orchestra.run.smithi171.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 196 kB/s | 33 kB 00:00 2024-04-26T04:32:16.813 INFO:teuthology.orchestra.run.smithi156.stdout:(11/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 28 MB/s | 16 MB 00:00 2024-04-26T04:32:16.850 INFO:teuthology.orchestra.run.smithi171.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 182 kB/s | 18 kB 00:00 2024-04-26T04:32:16.909 INFO:teuthology.orchestra.run.smithi171.stdout:(5/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 16 MB/s | 3.0 MB 00:00 2024-04-26T04:32:16.934 INFO:teuthology.orchestra.run.smithi171.stdout:(6/12): openblas-0.3.21-2.el9.x86_64.rpm 426 kB/s | 35 kB 00:00 2024-04-26T04:32:17.134 INFO:teuthology.orchestra.run.smithi171.stdout:(7/12): python3-devel-3.9.18-3.el9.x86_64.rpm 1.2 MB/s | 244 kB 00:00 2024-04-26T04:32:17.218 INFO:teuthology.orchestra.run.smithi171.stdout:(8/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 15 MB/s | 4.7 MB 00:00 2024-04-26T04:32:17.418 INFO:teuthology.orchestra.run.smithi171.stdout:(9/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 18 MB/s | 5.1 MB 00:00 2024-04-26T04:32:17.443 INFO:teuthology.orchestra.run.smithi171.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 1.1 MB/s | 250 kB 00:00 2024-04-26T04:32:18.077 INFO:teuthology.orchestra.run.smithi171.stdout:(11/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 25 MB/s | 16 MB 00:00 2024-04-26T04:32:20.227 INFO:teuthology.orchestra.run.smithi171.stdout:(12/12): ceph-mgr-diskprediction-local-17.2.7-9 1.8 MB/s | 7.4 MB 00:04 2024-04-26T04:32:20.230 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:20.231 INFO:teuthology.orchestra.run.smithi171.stdout:Total 8.7 MB/s | 38 MB 00:04 2024-04-26T04:32:20.587 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-04-26T04:32:20.614 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-04-26T04:32:20.614 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-04-26T04:32:21.025 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-04-26T04:32:21.025 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-04-26T04:32:21.780 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-04-26T04:32:21.840 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-04-26T04:32:21.884 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:32:21.982 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:32:22.034 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-04-26T04:32:22.155 INFO:teuthology.orchestra.run.smithi156.stdout:(12/12): ceph-mgr-diskprediction-local-17.2.7-9 1.1 MB/s | 7.4 MB 00:06 2024-04-26T04:32:22.158 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:22.158 INFO:teuthology.orchestra.run.smithi156.stdout:Total 5.1 MB/s | 38 MB 00:07 2024-04-26T04:32:22.215 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-26T04:32:22.242 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-26T04:32:22.319 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-26T04:32:22.516 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-04-26T04:32:22.542 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-04-26T04:32:22.543 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-04-26T04:32:22.948 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-04-26T04:32:22.949 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-04-26T04:32:22.975 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-26T04:32:23.089 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-26T04:32:23.708 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-04-26T04:32:23.776 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-04-26T04:32:23.819 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:32:23.921 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:32:23.962 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-04-26T04:32:24.148 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-26T04:32:24.170 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-26T04:32:24.246 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-26T04:32:24.334 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:32:24.888 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-26T04:32:25.008 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-26T04:32:25.372 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-26T04:32:25.408 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:32:26.231 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:32:27.276 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-26T04:32:27.320 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:32:27.810 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:32:27.810 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 1/12 2024-04-26T04:32:27.810 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:32:27.810 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:32:27.811 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-04-26T04:32:27.811 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-04-26T04:32:27.811 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-04-26T04:32:27.811 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-04-26T04:32:27.811 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-04-26T04:32:27.811 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-04-26T04:32:27.811 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:32:27.811 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-04-26T04:32:28.229 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-04-26T04:32:28.229 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:28.229 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-04-26T04:32:28.229 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:32:28.229 INFO:teuthology.orchestra.run.smithi171.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-26T04:32:28.229 INFO:teuthology.orchestra.run.smithi171.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-26T04:32:28.229 INFO:teuthology.orchestra.run.smithi171.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-26T04:32:28.229 INFO:teuthology.orchestra.run.smithi171.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-26T04:32:28.229 INFO:teuthology.orchestra.run.smithi171.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-26T04:32:28.229 INFO:teuthology.orchestra.run.smithi171.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-26T04:32:28.229 INFO:teuthology.orchestra.run.smithi171.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-26T04:32:28.229 INFO:teuthology.orchestra.run.smithi171.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-04-26T04:32:28.230 INFO:teuthology.orchestra.run.smithi171.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-26T04:32:28.230 INFO:teuthology.orchestra.run.smithi171.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-26T04:32:28.230 INFO:teuthology.orchestra.run.smithi171.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-26T04:32:28.230 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:28.230 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:28.627 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-mgr-rook 2024-04-26T04:32:29.148 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:19 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:29.275 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:29.276 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:29.277 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-04-26T04:32:29.277 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:29.277 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-04-26T04:32:29.277 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-rook noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 47 k 2024-04-26T04:32:29.277 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2024-04-26T04:32:29.277 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-26T04:32:29.277 INFO:teuthology.orchestra.run.smithi171.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-26T04:32:29.277 INFO:teuthology.orchestra.run.smithi171.stdout: python3-google-auth noarch 1:2.29.0-1.el9 epel 206 k 2024-04-26T04:32:29.277 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-26T04:32:29.277 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-26T04:32:29.277 INFO:teuthology.orchestra.run.smithi171.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-26T04:32:29.278 INFO:teuthology.orchestra.run.smithi171.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-26T04:32:29.278 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-26T04:32:29.278 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-26T04:32:29.278 INFO:teuthology.orchestra.run.smithi171.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-26T04:32:29.278 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-26T04:32:29.278 INFO:teuthology.orchestra.run.smithi171.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-26T04:32:29.278 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:29.278 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:32:29.278 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:29.278 INFO:teuthology.orchestra.run.smithi171.stdout:Install 13 Packages 2024-04-26T04:32:29.278 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:29.279 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 2.2 M 2024-04-26T04:32:29.279 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 26 M 2024-04-26T04:32:29.279 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-04-26T04:32:29.513 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:32:29.513 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 1/12 2024-04-26T04:32:29.514 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:32:29.514 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:32:29.514 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-04-26T04:32:29.514 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-04-26T04:32:29.514 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-04-26T04:32:29.514 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-04-26T04:32:29.514 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-04-26T04:32:29.514 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-04-26T04:32:29.514 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:32:29.514 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-04-26T04:32:29.709 INFO:teuthology.orchestra.run.smithi171.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 146 kB/s | 19 kB 00:00 2024-04-26T04:32:29.739 INFO:teuthology.orchestra.run.smithi171.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 164 kB/s | 26 kB 00:00 2024-04-26T04:32:29.793 INFO:teuthology.orchestra.run.smithi171.stdout:(3/13): ceph-mgr-rook-17.2.7-904.gd8c6b0a3.el9. 221 kB/s | 47 kB 00:00 2024-04-26T04:32:29.818 INFO:teuthology.orchestra.run.smithi171.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.0 MB/s | 222 kB 00:00 2024-04-26T04:32:29.843 INFO:teuthology.orchestra.run.smithi171.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.4 MB/s | 159 kB 00:00 2024-04-26T04:32:29.877 INFO:teuthology.orchestra.run.smithi171.stdout:(6/13): python3-pyasn1-modules-0.4.8-6.el9.noar 3.3 MB/s | 279 kB 00:00 2024-04-26T04:32:29.902 INFO:teuthology.orchestra.run.smithi171.stdout:(7/13): python3-requests-oauthlib-1.3.0-12.el9. 921 kB/s | 54 kB 00:00 2024-04-26T04:32:29.907 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-04-26T04:32:29.907 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:29.907 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-04-26T04:32:29.907 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:32:29.907 INFO:teuthology.orchestra.run.smithi156.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-26T04:32:29.907 INFO:teuthology.orchestra.run.smithi156.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-26T04:32:29.907 INFO:teuthology.orchestra.run.smithi156.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-26T04:32:29.907 INFO:teuthology.orchestra.run.smithi156.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-26T04:32:29.907 INFO:teuthology.orchestra.run.smithi156.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-26T04:32:29.908 INFO:teuthology.orchestra.run.smithi156.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-26T04:32:29.908 INFO:teuthology.orchestra.run.smithi156.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-26T04:32:29.908 INFO:teuthology.orchestra.run.smithi156.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-04-26T04:32:29.908 INFO:teuthology.orchestra.run.smithi156.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-26T04:32:29.908 INFO:teuthology.orchestra.run.smithi156.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-26T04:32:29.908 INFO:teuthology.orchestra.run.smithi156.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-26T04:32:29.908 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:29.908 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:30.255 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-mgr-rook 2024-04-26T04:32:30.261 INFO:teuthology.orchestra.run.smithi171.stdout:(8/13): python3-certifi-2021.10.8-2.el9.noarch. 39 kB/s | 15 kB 00:00 2024-04-26T04:32:30.295 INFO:teuthology.orchestra.run.smithi171.stdout:(9/13): python3-cachetools-4.2.4-1.el9.noarch.r 71 kB/s | 32 kB 00:00 2024-04-26T04:32:30.512 INFO:teuthology.orchestra.run.smithi171.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 273 kB/s | 59 kB 00:00 2024-04-26T04:32:30.537 INFO:teuthology.orchestra.run.smithi171.stdout:(11/13): python3-google-auth-2.29.0-1.el9.noarc 324 kB/s | 206 kB 00:00 2024-04-26T04:32:30.679 INFO:teuthology.orchestra.run.smithi171.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 538 kB/s | 90 kB 00:00 2024-04-26T04:32:30.796 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:22 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:30.821 INFO:teuthology.orchestra.run.smithi171.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 1.8 MB/s | 1.0 MB 00:00 2024-04-26T04:32:30.825 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:30.826 INFO:teuthology.orchestra.run.smithi171.stdout:Total 1.4 MB/s | 2.2 MB 00:01 2024-04-26T04:32:30.888 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-04-26T04:32:30.903 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-04-26T04:32:30.904 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-04-26T04:32:30.925 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:30.926 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:30.926 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-04-26T04:32:30.926 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:30.926 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-04-26T04:32:30.926 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-rook noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 47 k 2024-04-26T04:32:30.926 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2024-04-26T04:32:30.926 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-26T04:32:30.926 INFO:teuthology.orchestra.run.smithi156.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-26T04:32:30.927 INFO:teuthology.orchestra.run.smithi156.stdout: python3-google-auth noarch 1:2.29.0-1.el9 epel 206 k 2024-04-26T04:32:30.927 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-26T04:32:30.927 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-26T04:32:30.927 INFO:teuthology.orchestra.run.smithi156.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-26T04:32:30.927 INFO:teuthology.orchestra.run.smithi156.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-26T04:32:30.927 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-26T04:32:30.927 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-26T04:32:30.927 INFO:teuthology.orchestra.run.smithi156.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-26T04:32:30.927 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-26T04:32:30.927 INFO:teuthology.orchestra.run.smithi156.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-26T04:32:30.927 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:30.927 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:32:30.927 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:30.927 INFO:teuthology.orchestra.run.smithi156.stdout:Install 13 Packages 2024-04-26T04:32:30.928 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:30.928 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 2.2 M 2024-04-26T04:32:30.928 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 26 M 2024-04-26T04:32:30.928 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-04-26T04:32:30.987 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-04-26T04:32:30.989 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-04-26T04:32:31.238 INFO:teuthology.orchestra.run.smithi156.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 171 kB/s | 19 kB 00:00 2024-04-26T04:32:31.263 INFO:teuthology.orchestra.run.smithi156.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 187 kB/s | 26 kB 00:00 2024-04-26T04:32:31.314 INFO:teuthology.orchestra.run.smithi156.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.9 MB/s | 222 kB 00:00 2024-04-26T04:32:31.339 INFO:teuthology.orchestra.run.smithi156.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 2.1 MB/s | 159 kB 00:00 2024-04-26T04:32:31.364 INFO:teuthology.orchestra.run.smithi156.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 5.5 MB/s | 279 kB 00:00 2024-04-26T04:32:31.390 INFO:teuthology.orchestra.run.smithi156.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 1.1 MB/s | 54 kB 00:00 2024-04-26T04:32:31.448 INFO:teuthology.orchestra.run.smithi156.stdout:(7/13): ceph-mgr-rook-17.2.7-904.gd8c6b0a3.el9. 147 kB/s | 47 kB 00:00 2024-04-26T04:32:31.556 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-04-26T04:32:31.624 INFO:teuthology.orchestra.run.smithi156.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 124 kB/s | 32 kB 00:00 2024-04-26T04:32:31.649 INFO:teuthology.orchestra.run.smithi156.stdout:(9/13): python3-certifi-2021.10.8-2.el9.noarch. 58 kB/s | 15 kB 00:00 2024-04-26T04:32:31.746 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-26T04:32:31.816 INFO:teuthology.orchestra.run.smithi156.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 355 kB/s | 59 kB 00:00 2024-04-26T04:32:31.816 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-26T04:32:31.880 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-26T04:32:31.883 INFO:teuthology.orchestra.run.smithi156.stdout:(11/13): python3-google-auth-2.29.0-1.el9.noarc 473 kB/s | 206 kB 00:00 2024-04-26T04:32:31.917 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-26T04:32:31.958 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-26T04:32:32.008 INFO:teuthology.orchestra.run.smithi156.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 467 kB/s | 90 kB 00:00 2024-04-26T04:32:32.042 INFO:teuthology.orchestra.run.smithi156.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 2.4 MB/s | 1.0 MB 00:00 2024-04-26T04:32:32.043 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:32.044 INFO:teuthology.orchestra.run.smithi156.stdout:Total 2.0 MB/s | 2.2 MB 00:01 2024-04-26T04:32:32.053 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-26T04:32:32.105 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-04-26T04:32:32.120 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-04-26T04:32:32.120 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-04-26T04:32:32.153 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-google-auth-1:2.29.0-1.el9.noarch 7/13 2024-04-26T04:32:32.204 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-04-26T04:32:32.205 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-04-26T04:32:32.210 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-26T04:32:32.777 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-04-26T04:32:32.927 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-26T04:32:32.948 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-26T04:32:32.974 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-26T04:32:33.003 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-26T04:32:33.007 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-26T04:32:33.053 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-26T04:32:33.059 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-26T04:32:33.077 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:32:33.095 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-26T04:32:33.128 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-26T04:32:33.226 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-26T04:32:33.336 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-google-auth-1:2.29.0-1.el9.noarch 7/13 2024-04-26T04:32:33.397 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-26T04:32:33.814 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:32:33.815 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/13 2024-04-26T04:32:33.815 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-26T04:32:33.815 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-26T04:32:33.815 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-26T04:32:33.815 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-26T04:32:33.815 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-26T04:32:33.815 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-26T04:32:33.815 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-26T04:32:33.815 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-26T04:32:33.815 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-google-auth-1:2.29.0-1.el9.noarch 10/13 2024-04-26T04:32:33.815 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-26T04:32:33.815 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-26T04:32:34.125 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-26T04:32:34.125 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:34.125 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-04-26T04:32:34.125 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:32:34.125 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-26T04:32:34.125 INFO:teuthology.orchestra.run.smithi171.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-26T04:32:34.125 INFO:teuthology.orchestra.run.smithi171.stdout: python3-google-auth-1:2.29.0-1.el9.noarch 2024-04-26T04:32:34.126 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-26T04:32:34.126 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-26T04:32:34.126 INFO:teuthology.orchestra.run.smithi171.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-26T04:32:34.126 INFO:teuthology.orchestra.run.smithi171.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-26T04:32:34.126 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-26T04:32:34.126 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-26T04:32:34.126 INFO:teuthology.orchestra.run.smithi171.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-26T04:32:34.126 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-26T04:32:34.126 INFO:teuthology.orchestra.run.smithi171.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-26T04:32:34.127 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:34.127 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:34.196 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-26T04:32:34.252 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-26T04:32:34.285 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-26T04:32:34.330 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-26T04:32:34.355 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:32:34.376 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-mgr-cephadm 2024-04-26T04:32:34.891 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:24 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:35.011 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:35.012 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:35.012 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-04-26T04:32:35.012 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:35.012 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-04-26T04:32:35.012 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 123 k 2024-04-26T04:32:35.012 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2024-04-26T04:32:35.012 INFO:teuthology.orchestra.run.smithi171.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-26T04:32:35.012 INFO:teuthology.orchestra.run.smithi171.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-26T04:32:35.012 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-26T04:32:35.013 INFO:teuthology.orchestra.run.smithi171.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-26T04:32:35.013 INFO:teuthology.orchestra.run.smithi171.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-26T04:32:35.013 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:35.013 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:32:35.013 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:35.013 INFO:teuthology.orchestra.run.smithi171.stdout:Install 6 Packages 2024-04-26T04:32:35.013 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:35.013 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 7.0 M 2024-04-26T04:32:35.014 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 33 M 2024-04-26T04:32:35.014 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-04-26T04:32:35.125 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:32:35.126 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/13 2024-04-26T04:32:35.126 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-26T04:32:35.126 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-26T04:32:35.126 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-26T04:32:35.126 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-26T04:32:35.126 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-26T04:32:35.126 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-26T04:32:35.126 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-26T04:32:35.126 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-26T04:32:35.126 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-google-auth-1:2.29.0-1.el9.noarch 10/13 2024-04-26T04:32:35.126 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-26T04:32:35.127 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-26T04:32:35.436 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-26T04:32:35.436 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:35.436 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-04-26T04:32:35.436 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:32:35.436 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-26T04:32:35.436 INFO:teuthology.orchestra.run.smithi156.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-26T04:32:35.436 INFO:teuthology.orchestra.run.smithi156.stdout: python3-google-auth-1:2.29.0-1.el9.noarch 2024-04-26T04:32:35.436 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-26T04:32:35.436 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-26T04:32:35.437 INFO:teuthology.orchestra.run.smithi156.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-26T04:32:35.437 INFO:teuthology.orchestra.run.smithi156.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-26T04:32:35.437 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-26T04:32:35.437 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-26T04:32:35.437 INFO:teuthology.orchestra.run.smithi156.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-26T04:32:35.437 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-26T04:32:35.437 INFO:teuthology.orchestra.run.smithi156.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-26T04:32:35.437 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:35.437 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:35.524 INFO:teuthology.orchestra.run.smithi171.stdout:(1/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 1.2 MB/s | 248 kB 00:00 2024-04-26T04:32:35.691 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-mgr-cephadm 2024-04-26T04:32:35.707 INFO:teuthology.orchestra.run.smithi171.stdout:(2/6): python3-babel-2.9.1-2.el9.noarch.rpm 15 MB/s | 6.0 MB 00:00 2024-04-26T04:32:35.791 INFO:teuthology.orchestra.run.smithi171.stdout:(3/6): ceph-mgr-cephadm-17.2.7-904.gd8c6b0a3.el 263 kB/s | 123 kB 00:00 2024-04-26T04:32:35.866 INFO:teuthology.orchestra.run.smithi171.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 364 kB/s | 58 kB 00:00 2024-04-26T04:32:35.950 INFO:teuthology.orchestra.run.smithi171.stdout:(5/6): python3-typing-extensions-4.4.0-2.el9.no 321 kB/s | 51 kB 00:00 2024-04-26T04:32:35.975 INFO:teuthology.orchestra.run.smithi171.stdout:(6/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 1.2 MB/s | 548 kB 00:00 2024-04-26T04:32:35.977 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:35.977 INFO:teuthology.orchestra.run.smithi171.stdout:Total 7.2 MB/s | 7.0 MB 00:00 2024-04-26T04:32:36.049 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-04-26T04:32:36.060 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-04-26T04:32:36.060 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-04-26T04:32:36.157 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-04-26T04:32:36.157 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-04-26T04:32:36.207 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:28 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:36.328 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:36.328 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:36.329 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-04-26T04:32:36.329 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:36.329 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-04-26T04:32:36.329 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 123 k 2024-04-26T04:32:36.329 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2024-04-26T04:32:36.329 INFO:teuthology.orchestra.run.smithi156.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-26T04:32:36.329 INFO:teuthology.orchestra.run.smithi156.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-26T04:32:36.329 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-26T04:32:36.329 INFO:teuthology.orchestra.run.smithi156.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-26T04:32:36.329 INFO:teuthology.orchestra.run.smithi156.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-26T04:32:36.329 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:36.329 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:32:36.329 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:36.330 INFO:teuthology.orchestra.run.smithi156.stdout:Install 6 Packages 2024-04-26T04:32:36.330 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:36.330 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 7.0 M 2024-04-26T04:32:36.330 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 33 M 2024-04-26T04:32:36.330 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-04-26T04:32:36.467 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-04-26T04:32:36.641 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-26T04:32:36.694 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-26T04:32:36.920 INFO:teuthology.orchestra.run.smithi156.stdout:(1/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 1.4 MB/s | 248 kB 00:00 2024-04-26T04:32:37.012 INFO:teuthology.orchestra.run.smithi156.stdout:(2/6): ceph-mgr-cephadm-17.2.7-904.gd8c6b0a3.el 466 kB/s | 123 kB 00:00 2024-04-26T04:32:37.179 INFO:teuthology.orchestra.run.smithi156.stdout:(3/6): python3-babel-2.9.1-2.el9.noarch.rpm 14 MB/s | 6.0 MB 00:00 2024-04-26T04:32:37.221 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-26T04:32:37.305 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-26T04:32:37.360 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-26T04:32:37.389 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:32:37.563 INFO:teuthology.orchestra.run.smithi156.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 110 kB/s | 58 kB 00:00 2024-04-26T04:32:37.621 INFO:teuthology.orchestra.run.smithi156.stdout:(5/6): python3-typing-extensions-4.4.0-2.el9.no 122 kB/s | 51 kB 00:00 2024-04-26T04:32:37.688 INFO:teuthology.orchestra.run.smithi156.stdout:(6/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 715 kB/s | 548 kB 00:00 2024-04-26T04:32:37.693 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:37.693 INFO:teuthology.orchestra.run.smithi156.stdout:Total 5.1 MB/s | 7.0 MB 00:01 2024-04-26T04:32:37.764 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-04-26T04:32:37.775 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-04-26T04:32:37.775 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-04-26T04:32:37.871 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-04-26T04:32:37.872 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-04-26T04:32:38.189 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-04-26T04:32:38.218 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:32:38.218 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/6 2024-04-26T04:32:38.218 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-26T04:32:38.218 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-26T04:32:38.219 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-26T04:32:38.219 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-26T04:32:38.353 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-26T04:32:38.406 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-26T04:32:38.464 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-26T04:32:38.464 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:38.464 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-04-26T04:32:38.465 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:32:38.465 INFO:teuthology.orchestra.run.smithi171.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-26T04:32:38.465 INFO:teuthology.orchestra.run.smithi171.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-26T04:32:38.465 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-26T04:32:38.465 INFO:teuthology.orchestra.run.smithi171.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-26T04:32:38.465 INFO:teuthology.orchestra.run.smithi171.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-26T04:32:38.465 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:38.465 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:38.705 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-fuse 2024-04-26T04:32:38.931 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-26T04:32:39.009 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-26T04:32:39.073 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-26T04:32:39.102 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:32:39.226 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:29 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:39.347 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:39.348 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:39.348 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-04-26T04:32:39.348 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:39.348 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-04-26T04:32:39.348 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 812 k 2024-04-26T04:32:39.349 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2024-04-26T04:32:39.349 INFO:teuthology.orchestra.run.smithi171.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-26T04:32:39.349 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:39.349 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:32:39.349 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:39.349 INFO:teuthology.orchestra.run.smithi171.stdout:Install 2 Packages 2024-04-26T04:32:39.349 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:39.350 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 892 k 2024-04-26T04:32:39.350 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 2.6 M 2024-04-26T04:32:39.350 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-04-26T04:32:39.677 INFO:teuthology.orchestra.run.smithi171.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 391 kB/s | 80 kB 00:00 2024-04-26T04:32:39.926 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:32:39.926 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/6 2024-04-26T04:32:39.926 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-26T04:32:39.926 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-26T04:32:39.926 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-26T04:32:39.926 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-26T04:32:40.169 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-26T04:32:40.169 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:40.169 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-04-26T04:32:40.169 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:32:40.169 INFO:teuthology.orchestra.run.smithi156.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-26T04:32:40.169 INFO:teuthology.orchestra.run.smithi156.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-26T04:32:40.169 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-26T04:32:40.169 INFO:teuthology.orchestra.run.smithi156.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-26T04:32:40.169 INFO:teuthology.orchestra.run.smithi156.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-26T04:32:40.169 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:40.169 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:40.318 INFO:teuthology.orchestra.run.smithi171.stdout:(2/2): ceph-fuse-17.2.7-904.gd8c6b0a3.el9.x86_6 958 kB/s | 812 kB 00:00 2024-04-26T04:32:40.320 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:40.320 INFO:teuthology.orchestra.run.smithi171.stdout:Total 918 kB/s | 892 kB 00:00 2024-04-26T04:32:40.332 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-04-26T04:32:40.353 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-04-26T04:32:40.353 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-04-26T04:32:40.413 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-fuse 2024-04-26T04:32:40.430 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-04-26T04:32:40.430 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-04-26T04:32:40.611 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-04-26T04:32:40.756 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-26T04:32:40.823 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:32:40.930 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:32 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:41.051 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:41.052 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:41.052 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-04-26T04:32:41.052 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:41.052 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-04-26T04:32:41.052 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 812 k 2024-04-26T04:32:41.052 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2024-04-26T04:32:41.052 INFO:teuthology.orchestra.run.smithi156.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-26T04:32:41.052 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:41.052 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:32:41.052 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:41.052 INFO:teuthology.orchestra.run.smithi156.stdout:Install 2 Packages 2024-04-26T04:32:41.053 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:41.053 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 892 k 2024-04-26T04:32:41.053 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 2.6 M 2024-04-26T04:32:41.053 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-04-26T04:32:41.422 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:32:41.422 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/2 2024-04-26T04:32:41.457 INFO:teuthology.orchestra.run.smithi156.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 630 kB/s | 80 kB 00:00 2024-04-26T04:32:41.659 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-26T04:32:41.659 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:41.659 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-04-26T04:32:41.659 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-26T04:32:41.659 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:41.659 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:41.824 INFO:teuthology.orchestra.run.smithi156.stdout:(2/2): ceph-fuse-17.2.7-904.gd8c6b0a3.el9.x86_6 1.6 MB/s | 812 kB 00:00 2024-04-26T04:32:41.826 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:41.826 INFO:teuthology.orchestra.run.smithi156.stdout:Total 1.1 MB/s | 892 kB 00:00 2024-04-26T04:32:41.836 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-04-26T04:32:41.857 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-04-26T04:32:41.857 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-04-26T04:32:41.929 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install librados-devel 2024-04-26T04:32:41.933 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-04-26T04:32:41.933 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-04-26T04:32:42.122 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-04-26T04:32:42.246 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-26T04:32:42.320 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:32:42.444 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:32 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:42.566 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:42.567 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:42.567 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repo Size 2024-04-26T04:32:42.567 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:42.567 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-04-26T04:32:42.567 INFO:teuthology.orchestra.run.smithi171.stdout: librados-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 126 k 2024-04-26T04:32:42.567 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:42.567 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:32:42.568 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:42.568 INFO:teuthology.orchestra.run.smithi171.stdout:Install 1 Package 2024-04-26T04:32:42.568 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:42.568 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 126 k 2024-04-26T04:32:42.568 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 448 k 2024-04-26T04:32:42.568 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-04-26T04:32:42.806 INFO:teuthology.orchestra.run.smithi171.stdout:librados-devel-17.2.7-904.gd8c6b0a3.el9.x86_64. 530 kB/s | 126 kB 00:00 2024-04-26T04:32:42.807 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:42.807 INFO:teuthology.orchestra.run.smithi171.stdout:Total 526 kB/s | 126 kB 00:00 2024-04-26T04:32:42.807 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-04-26T04:32:42.813 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-04-26T04:32:42.813 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-04-26T04:32:42.876 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-04-26T04:32:42.876 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-04-26T04:32:42.914 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:32:42.914 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/2 2024-04-26T04:32:43.031 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-04-26T04:32:43.142 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-26T04:32:43.142 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:43.142 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-04-26T04:32:43.142 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-26T04:32:43.142 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:43.142 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:43.160 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:43.413 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install librados-devel 2024-04-26T04:32:43.495 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:43.750 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:43.750 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:43.750 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-04-26T04:32:43.750 INFO:teuthology.orchestra.run.smithi171.stdout: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:43.750 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:43.750 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:43.926 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:35 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:43.974 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install libcephfs2 2024-04-26T04:32:44.048 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:44.049 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:44.049 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repo Size 2024-04-26T04:32:44.049 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:44.049 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-04-26T04:32:44.049 INFO:teuthology.orchestra.run.smithi156.stdout: librados-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 126 k 2024-04-26T04:32:44.049 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:44.049 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:32:44.049 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:44.049 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2024-04-26T04:32:44.049 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:44.049 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 126 k 2024-04-26T04:32:44.049 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 448 k 2024-04-26T04:32:44.050 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-04-26T04:32:44.484 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:34 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:44.565 INFO:teuthology.orchestra.run.smithi171.stdout:Package libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:44.570 INFO:teuthology.orchestra.run.smithi156.stdout:librados-devel-17.2.7-904.gd8c6b0a3.el9.x86_64. 243 kB/s | 126 kB 00:00 2024-04-26T04:32:44.570 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:44.571 INFO:teuthology.orchestra.run.smithi156.stdout:Total 242 kB/s | 126 kB 00:00 2024-04-26T04:32:44.571 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-04-26T04:32:44.577 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-04-26T04:32:44.577 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-04-26T04:32:44.606 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:44.607 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-04-26T04:32:44.607 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:44.639 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-04-26T04:32:44.640 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-04-26T04:32:44.693 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install libcephfs-devel 2024-04-26T04:32:44.793 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-04-26T04:32:44.917 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:45.237 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:35 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:45.341 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:45.356 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:45.357 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:45.357 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repo Size 2024-04-26T04:32:45.357 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:45.357 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-04-26T04:32:45.357 INFO:teuthology.orchestra.run.smithi171.stdout: libcephfs-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:32:45.357 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:45.357 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:32:45.357 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:45.357 INFO:teuthology.orchestra.run.smithi171.stdout:Install 1 Package 2024-04-26T04:32:45.357 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:45.358 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 25 k 2024-04-26T04:32:45.358 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 110 k 2024-04-26T04:32:45.358 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-04-26T04:32:45.526 INFO:teuthology.orchestra.run.smithi171.stdout:libcephfs-devel-17.2.7-904.gd8c6b0a3.el9.x86_64 148 kB/s | 25 kB 00:00 2024-04-26T04:32:45.527 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:45.527 INFO:teuthology.orchestra.run.smithi171.stdout:Total 147 kB/s | 25 kB 00:00 2024-04-26T04:32:45.527 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-04-26T04:32:45.530 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-04-26T04:32:45.531 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-04-26T04:32:45.532 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-04-26T04:32:45.532 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-04-26T04:32:45.579 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:45.579 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:45.579 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-04-26T04:32:45.579 INFO:teuthology.orchestra.run.smithi156.stdout: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:45.579 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:45.579 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:45.621 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-04-26T04:32:45.739 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:45.812 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install libcephfs2 2024-04-26T04:32:46.080 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:46.280 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:46.280 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:46.280 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-04-26T04:32:46.280 INFO:teuthology.orchestra.run.smithi171.stdout: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:46.280 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:46.280 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:46.327 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:38 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:46.406 INFO:teuthology.orchestra.run.smithi156.stdout:Package libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:46.448 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:46.449 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-04-26T04:32:46.449 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:46.464 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install librados2 2024-04-26T04:32:46.534 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install libcephfs-devel 2024-04-26T04:32:46.971 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:36 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:47.048 INFO:teuthology.orchestra.run.smithi171.stdout:Package librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:47.080 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:39 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:47.090 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:47.091 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-04-26T04:32:47.091 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:47.163 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install librbd1 2024-04-26T04:32:47.201 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:47.201 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:47.201 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repo Size 2024-04-26T04:32:47.201 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:47.201 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-04-26T04:32:47.201 INFO:teuthology.orchestra.run.smithi156.stdout: libcephfs-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:32:47.201 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:47.201 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:32:47.201 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:47.201 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2024-04-26T04:32:47.201 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:47.202 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 25 k 2024-04-26T04:32:47.202 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 110 k 2024-04-26T04:32:47.202 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-04-26T04:32:47.366 INFO:teuthology.orchestra.run.smithi156.stdout:libcephfs-devel-17.2.7-904.gd8c6b0a3.el9.x86_64 152 kB/s | 25 kB 00:00 2024-04-26T04:32:47.367 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:47.367 INFO:teuthology.orchestra.run.smithi156.stdout:Total 151 kB/s | 25 kB 00:00 2024-04-26T04:32:47.367 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-04-26T04:32:47.369 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-04-26T04:32:47.369 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-04-26T04:32:47.370 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-04-26T04:32:47.371 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-04-26T04:32:47.446 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-04-26T04:32:47.551 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:47.682 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:37 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:47.762 INFO:teuthology.orchestra.run.smithi171.stdout:Package librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:47.803 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:47.804 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-04-26T04:32:47.804 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:47.877 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install python3-rados 2024-04-26T04:32:47.886 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:48.085 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:48.085 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:48.086 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-04-26T04:32:48.086 INFO:teuthology.orchestra.run.smithi156.stdout: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:48.086 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:48.086 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:48.283 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install librados2 2024-04-26T04:32:48.386 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:38 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:48.464 INFO:teuthology.orchestra.run.smithi171.stdout:Package python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:48.505 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:48.506 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-04-26T04:32:48.506 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:48.576 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install python3-rgw 2024-04-26T04:32:48.790 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:40 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:48.869 INFO:teuthology.orchestra.run.smithi156.stdout:Package librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:48.909 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:48.910 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-04-26T04:32:48.910 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:48.992 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install librbd1 2024-04-26T04:32:49.091 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:39 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:49.169 INFO:teuthology.orchestra.run.smithi171.stdout:Package python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:49.211 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:49.212 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-04-26T04:32:49.212 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:49.288 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install python3-cephfs 2024-04-26T04:32:49.500 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:41 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:49.579 INFO:teuthology.orchestra.run.smithi156.stdout:Package librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:49.620 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:49.620 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-04-26T04:32:49.621 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:49.700 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install python3-rados 2024-04-26T04:32:49.804 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:39 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:49.884 INFO:teuthology.orchestra.run.smithi171.stdout:Package python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:49.924 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:49.925 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-04-26T04:32:49.925 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:49.992 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install python3-rbd 2024-04-26T04:32:50.206 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:42 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:50.287 INFO:teuthology.orchestra.run.smithi156.stdout:Package python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:50.327 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:50.328 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-04-26T04:32:50.328 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:50.405 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install python3-rgw 2024-04-26T04:32:50.502 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:40 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:50.582 INFO:teuthology.orchestra.run.smithi171.stdout:Package python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:50.622 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:50.623 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-04-26T04:32:50.623 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:50.695 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install rbd-fuse 2024-04-26T04:32:50.919 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:42 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:50.999 INFO:teuthology.orchestra.run.smithi156.stdout:Package python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:51.039 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:51.040 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-04-26T04:32:51.040 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:51.117 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install python3-cephfs 2024-04-26T04:32:51.203 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:41 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:51.323 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:51.323 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:51.323 INFO:teuthology.orchestra.run.smithi171.stdout: Package Architecture Version Repository Size 2024-04-26T04:32:51.323 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:51.323 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-04-26T04:32:51.323 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 82 k 2024-04-26T04:32:51.324 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:51.324 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:32:51.324 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:51.324 INFO:teuthology.orchestra.run.smithi171.stdout:Install 1 Package 2024-04-26T04:32:51.324 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:51.324 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 82 k 2024-04-26T04:32:51.324 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 222 k 2024-04-26T04:32:51.325 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-04-26T04:32:51.628 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:43 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:51.707 INFO:teuthology.orchestra.run.smithi156.stdout:Package python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:51.747 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:51.748 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-04-26T04:32:51.748 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:51.779 INFO:teuthology.orchestra.run.smithi171.stdout:rbd-fuse-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 180 kB/s | 82 kB 00:00 2024-04-26T04:32:51.779 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:51.779 INFO:teuthology.orchestra.run.smithi171.stdout:Total 180 kB/s | 82 kB 00:00 2024-04-26T04:32:51.780 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-04-26T04:32:51.786 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-04-26T04:32:51.786 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-04-26T04:32:51.825 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install python3-rbd 2024-04-26T04:32:51.848 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-04-26T04:32:51.849 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-04-26T04:32:51.990 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-04-26T04:32:52.090 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:52.332 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:44 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:52.411 INFO:teuthology.orchestra.run.smithi156.stdout:Package python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:32:52.420 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:52.451 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:52.452 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-04-26T04:32:52.452 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:52.532 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install rbd-fuse 2024-04-26T04:32:52.640 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:52.640 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:52.640 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-04-26T04:32:52.640 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:52.640 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:52.640 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:52.896 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install rbd-mirror 2024-04-26T04:32:53.043 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:45 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:53.164 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:53.165 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:53.165 INFO:teuthology.orchestra.run.smithi156.stdout: Package Architecture Version Repository Size 2024-04-26T04:32:53.165 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:53.165 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-04-26T04:32:53.165 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 82 k 2024-04-26T04:32:53.165 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:53.165 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:32:53.165 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:53.165 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2024-04-26T04:32:53.165 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:53.165 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 82 k 2024-04-26T04:32:53.165 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 222 k 2024-04-26T04:32:53.166 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-04-26T04:32:53.409 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:43 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:53.493 INFO:teuthology.orchestra.run.smithi156.stdout:rbd-fuse-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 250 kB/s | 82 kB 00:00 2024-04-26T04:32:53.493 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:53.494 INFO:teuthology.orchestra.run.smithi156.stdout:Total 249 kB/s | 82 kB 00:00 2024-04-26T04:32:53.494 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-04-26T04:32:53.501 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-04-26T04:32:53.501 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-04-26T04:32:53.529 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:53.530 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:53.530 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repo Size 2024-04-26T04:32:53.530 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:53.530 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-04-26T04:32:53.530 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-mirror x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:32:53.530 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:53.530 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:32:53.530 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:53.530 INFO:teuthology.orchestra.run.smithi171.stdout:Install 1 Package 2024-04-26T04:32:53.530 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:53.531 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 2.9 M 2024-04-26T04:32:53.531 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 12 M 2024-04-26T04:32:53.531 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-04-26T04:32:53.562 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-04-26T04:32:53.563 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-04-26T04:32:53.701 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-04-26T04:32:53.805 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:54.136 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:54.398 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:54.398 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:54.398 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-04-26T04:32:54.398 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:54.398 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:54.398 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:54.689 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install rbd-mirror 2024-04-26T04:32:55.007 INFO:teuthology.orchestra.run.smithi171.stdout:rbd-mirror-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 2.0 MB/s | 2.9 MB 00:01 2024-04-26T04:32:55.008 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:55.008 INFO:teuthology.orchestra.run.smithi171.stdout:Total 2.0 MB/s | 2.9 MB 00:01 2024-04-26T04:32:55.008 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-04-26T04:32:55.019 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-04-26T04:32:55.019 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-04-26T04:32:55.101 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-04-26T04:32:55.102 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-04-26T04:32:55.209 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:47 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:55.331 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:55.403 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:55.403 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repo Size 2024-04-26T04:32:55.404 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:55.404 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-04-26T04:32:55.404 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-mirror x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:32:55.404 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:55.404 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:32:55.404 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:55.404 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2024-04-26T04:32:55.404 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:55.404 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 2.9 M 2024-04-26T04:32:55.404 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 12 M 2024-04-26T04:32:55.404 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-04-26T04:32:55.518 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-04-26T04:32:55.582 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:55.606 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:55.606 INFO:teuthology.orchestra.run.smithi171.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:32:55.607 INFO:teuthology.orchestra.run.smithi171.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-26T04:32:55.607 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:32:55.607 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:32:55.607 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:56.354 INFO:teuthology.orchestra.run.smithi156.stdout:rbd-mirror-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 2.9 MB/s | 2.9 MB 00:01 2024-04-26T04:32:56.355 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:56.355 INFO:teuthology.orchestra.run.smithi156.stdout:Total 2.9 MB/s | 2.9 MB 00:01 2024-04-26T04:32:56.355 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-04-26T04:32:56.364 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-04-26T04:32:56.365 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-04-26T04:32:56.444 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-04-26T04:32:56.444 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-04-26T04:32:56.622 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:56.622 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:56.622 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-04-26T04:32:56.622 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:56.622 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:56.622 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:56.842 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install rbd-nbd 2024-04-26T04:32:56.872 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-04-26T04:32:56.930 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:56.954 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:56.954 INFO:teuthology.orchestra.run.smithi156.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:32:56.954 INFO:teuthology.orchestra.run.smithi156.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-26T04:32:56.954 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:32:56.955 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:32:56.955 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:57.364 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:47 ago on Fri 26 Apr 2024 04:31:10 AM UTC. 2024-04-26T04:32:57.486 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-04-26T04:32:57.486 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:57.486 INFO:teuthology.orchestra.run.smithi171.stdout: Package Architecture Version Repository Size 2024-04-26T04:32:57.486 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:57.486 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-04-26T04:32:57.486 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-nbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 165 k 2024-04-26T04:32:57.486 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:57.486 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-04-26T04:32:57.487 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-04-26T04:32:57.487 INFO:teuthology.orchestra.run.smithi171.stdout:Install 1 Package 2024-04-26T04:32:57.487 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:57.487 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 165 k 2024-04-26T04:32:57.487 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 477 k 2024-04-26T04:32:57.487 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-04-26T04:32:57.859 INFO:teuthology.orchestra.run.smithi171.stdout:rbd-nbd-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 443 kB/s | 165 kB 00:00 2024-04-26T04:32:57.860 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:57.860 INFO:teuthology.orchestra.run.smithi171.stdout:Total 441 kB/s | 165 kB 00:00 2024-04-26T04:32:57.860 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-04-26T04:32:57.868 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-04-26T04:32:57.868 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-04-26T04:32:57.918 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:57.918 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:57.918 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-04-26T04:32:57.918 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:57.918 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:57.918 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:32:57.930 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-04-26T04:32:57.930 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-04-26T04:32:58.081 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-04-26T04:32:58.158 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install rbd-nbd 2024-04-26T04:32:58.188 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:58.559 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:58.679 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:50 ago on Fri 26 Apr 2024 04:31:08 AM UTC. 2024-04-26T04:32:58.798 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-04-26T04:32:58.799 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:58.799 INFO:teuthology.orchestra.run.smithi156.stdout: Package Architecture Version Repository Size 2024-04-26T04:32:58.799 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:58.799 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-04-26T04:32:58.799 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-nbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 165 k 2024-04-26T04:32:58.800 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:58.800 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-04-26T04:32:58.800 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-04-26T04:32:58.800 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2024-04-26T04:32:58.800 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:32:58.800 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 165 k 2024-04-26T04:32:58.800 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 477 k 2024-04-26T04:32:58.800 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-04-26T04:32:58.815 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:58.815 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:58.815 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-04-26T04:32:58.815 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:32:58.815 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:32:58.815 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-04-26T04:32:59.101 DEBUG:teuthology.parallel:result is None 2024-04-26T04:32:59.149 INFO:teuthology.orchestra.run.smithi156.stdout:rbd-nbd-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 473 kB/s | 165 kB 00:00 2024-04-26T04:32:59.150 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:59.150 INFO:teuthology.orchestra.run.smithi156.stdout:Total 471 kB/s | 165 kB 00:00 2024-04-26T04:32:59.150 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-04-26T04:32:59.158 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-04-26T04:32:59.158 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-04-26T04:32:59.220 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-04-26T04:32:59.220 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-04-26T04:32:59.376 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-04-26T04:32:59.460 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:32:59.822 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:33:00.093 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:33:00.093 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:33:00.093 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-04-26T04:33:00.093 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:33:00.093 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:33:00.093 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-04-26T04:33:00.358 DEBUG:teuthology.parallel:result is None 2024-04-26T04:33:00.359 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:33:00.359 INFO:teuthology.packaging:ref: None 2024-04-26T04:33:00.359 INFO:teuthology.packaging:tag: None 2024-04-26T04:33:00.359 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:33:00.359 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:33:00.359 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:33:00.582 DEBUG:teuthology.orchestra.run.smithi156:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-26T04:33:00.602 INFO:teuthology.orchestra.run.smithi156.stdout:17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:33:00.603 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:33:00.603 INFO:teuthology.task.install:The correct ceph version 17.2.7-904.gd8c6b0a3 is installed. 2024-04-26T04:33:00.604 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:33:00.605 INFO:teuthology.packaging:ref: None 2024-04-26T04:33:00.605 INFO:teuthology.packaging:tag: None 2024-04-26T04:33:00.605 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:33:00.605 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:33:00.605 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:33:00.975 DEBUG:teuthology.orchestra.run.smithi171:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-26T04:33:01.002 INFO:teuthology.orchestra.run.smithi171.stdout:17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:33:01.002 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:33:01.002 INFO:teuthology.task.install:The correct ceph version 17.2.7-904.gd8c6b0a3 is installed. 2024-04-26T04:33:01.004 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-04-26T04:33:01.004 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-04-26T04:33:01.004 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-26T04:33:01.028 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-04-26T04:33:01.029 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-26T04:33:01.071 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-04-26T04:33:01.071 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-04-26T04:33:01.071 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/usr/bin/daemon-helper 2024-04-26T04:33:01.121 DEBUG:teuthology.orchestra.run.smithi156:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-26T04:33:01.185 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-04-26T04:33:01.185 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/usr/bin/daemon-helper 2024-04-26T04:33:01.214 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-26T04:33:01.280 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-04-26T04:33:01.281 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-04-26T04:33:01.281 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-26T04:33:01.305 DEBUG:teuthology.orchestra.run.smithi156:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-26T04:33:01.369 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-04-26T04:33:01.369 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-26T04:33:01.396 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-26T04:33:01.463 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-04-26T04:33:01.463 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-04-26T04:33:01.463 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/usr/bin/stdin-killer 2024-04-26T04:33:01.488 DEBUG:teuthology.orchestra.run.smithi156:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-26T04:33:01.552 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-04-26T04:33:01.552 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/usr/bin/stdin-killer 2024-04-26T04:33:01.579 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-26T04:33:01.646 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-26T04:33:01.749 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a'} 2024-04-26T04:33:01.749 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-04-26T04:33:01.749 INFO:tasks.cephadm:Cluster fsid is 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:33:01.750 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-26T04:33:01.750 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.156', 'mon.c': '[v2:172.21.15.156:3301,v1:172.21.15.156:6790]', 'mon.b': '172.21.15.171'} 2024-04-26T04:33:01.750 INFO:tasks.cephadm:First mon is mon.a on smithi156 2024-04-26T04:33:01.750 INFO:tasks.cephadm:First mgr is y 2024-04-26T04:33:01.750 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-26T04:33:01.750 DEBUG:teuthology.orchestra.run.smithi156:> sudo hostname $(hostname -s) 2024-04-26T04:33:01.775 DEBUG:teuthology.orchestra.run.smithi171:> sudo hostname $(hostname -s) 2024-04-26T04:33:01.802 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-04-26T04:33:01.803 DEBUG:teuthology.orchestra.run.smithi156:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-26T04:33:02.113 INFO:teuthology.orchestra.run.smithi156.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 26 04:33 /home/ubuntu/cephtest/cephadm 2024-04-26T04:33:02.114 DEBUG:teuthology.orchestra.run.smithi171:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-26T04:33:02.419 INFO:teuthology.orchestra.run.smithi171.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 26 04:33 /home/ubuntu/cephtest/cephadm 2024-04-26T04:33:02.420 DEBUG:teuthology.orchestra.run.smithi156:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-26T04:33:02.435 DEBUG:teuthology.orchestra.run.smithi171:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-26T04:33:02.464 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-04-26T04:33:02.464 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-26T04:33:02.478 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-26T04:33:04.008 INFO:teuthology.orchestra.run.smithi156.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:33:04.047 INFO:teuthology.orchestra.run.smithi171.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:33:35.563 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-04-26T04:33:35.563 INFO:teuthology.orchestra.run.smithi171.stdout: "ceph_version": "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)", 2024-04-26T04:33:35.563 INFO:teuthology.orchestra.run.smithi171.stdout: "image_id": "0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43", 2024-04-26T04:33:35.564 INFO:teuthology.orchestra.run.smithi171.stdout: "repo_digests": [ 2024-04-26T04:33:35.564 INFO:teuthology.orchestra.run.smithi171.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17" 2024-04-26T04:33:35.564 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-04-26T04:33:35.564 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-04-26T04:33:37.184 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2024-04-26T04:33:37.185 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph_version": "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)", 2024-04-26T04:33:37.185 INFO:teuthology.orchestra.run.smithi156.stdout: "image_id": "0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43", 2024-04-26T04:33:37.185 INFO:teuthology.orchestra.run.smithi156.stdout: "repo_digests": [ 2024-04-26T04:33:37.185 INFO:teuthology.orchestra.run.smithi156.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17" 2024-04-26T04:33:37.185 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2024-04-26T04:33:37.185 INFO:teuthology.orchestra.run.smithi156.stdout:} 2024-04-26T04:33:37.203 DEBUG:teuthology.orchestra.run.smithi156:> sudo mkdir -p /etc/ceph 2024-04-26T04:33:37.248 DEBUG:teuthology.orchestra.run.smithi171:> sudo mkdir -p /etc/ceph 2024-04-26T04:33:37.282 DEBUG:teuthology.orchestra.run.smithi156:> sudo chmod 777 /etc/ceph 2024-04-26T04:33:37.311 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod 777 /etc/ceph 2024-04-26T04:33:37.348 INFO:tasks.cephadm:Writing seed config... 2024-04-26T04:33:37.350 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-04-26T04:33:37.350 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-04-26T04:33:37.350 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-26T04:33:37.350 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-26T04:33:37.350 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-26T04:33:37.350 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-04-26T04:33:37.350 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-26T04:33:37.350 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-26T04:33:37.350 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-26T04:33:37.350 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-26T04:33:37.350 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-26T04:33:37.351 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-04-26T04:33:37.351 DEBUG:teuthology.orchestra.run.smithi156:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-26T04:33:37.368 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 110027f2-0386-11ef-bc93-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-26T04:33:37.368 DEBUG:teuthology.orchestra.run.smithi156:mon.a> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.a.service 2024-04-26T04:33:37.410 DEBUG:teuthology.orchestra.run.smithi156:mgr.y> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.y.service 2024-04-26T04:33:37.452 INFO:tasks.cephadm:Bootstrapping... 2024-04-26T04:33:37.453 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 110027f2-0386-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.156 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:33:37.653 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:33:37.653 INFO:teuthology.orchestra.run.smithi156.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '110027f2-0386-11ef-bc93-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.156', '--skip-admin-label'] 2024-04-26T04:33:37.679 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-26T04:33:37.680 INFO:teuthology.orchestra.run.smithi156.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-04-26T04:33:37.680 INFO:teuthology.orchestra.run.smithi156.stdout:Verifying podman|docker is present... 2024-04-26T04:33:37.704 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-26T04:33:37.704 INFO:teuthology.orchestra.run.smithi156.stdout:Verifying lvm2 is present... 2024-04-26T04:33:37.704 INFO:teuthology.orchestra.run.smithi156.stdout:Verifying time synchronization is in place... 2024-04-26T04:33:37.711 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-26T04:33:37.711 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-26T04:33:37.717 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-26T04:33:37.717 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout inactive 2024-04-26T04:33:37.724 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout enabled 2024-04-26T04:33:37.730 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout active 2024-04-26T04:33:37.730 INFO:teuthology.orchestra.run.smithi156.stdout:Unit chronyd.service is enabled and running 2024-04-26T04:33:37.730 INFO:teuthology.orchestra.run.smithi156.stdout:Repeating the final host check... 2024-04-26T04:33:37.753 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-26T04:33:37.753 INFO:teuthology.orchestra.run.smithi156.stdout:podman (/bin/podman) version 4.9.4 is present 2024-04-26T04:33:37.754 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl is present 2024-04-26T04:33:37.754 INFO:teuthology.orchestra.run.smithi156.stdout:lvcreate is present 2024-04-26T04:33:37.760 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-26T04:33:37.760 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-26T04:33:37.766 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-26T04:33:37.766 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout inactive 2024-04-26T04:33:37.772 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout enabled 2024-04-26T04:33:37.779 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout active 2024-04-26T04:33:37.779 INFO:teuthology.orchestra.run.smithi156.stdout:Unit chronyd.service is enabled and running 2024-04-26T04:33:37.779 INFO:teuthology.orchestra.run.smithi156.stdout:Host looks OK 2024-04-26T04:33:37.779 INFO:teuthology.orchestra.run.smithi156.stdout:Cluster fsid: 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:33:37.779 INFO:teuthology.orchestra.run.smithi156.stdout:Acquiring lock 140481464967376 on /run/cephadm/110027f2-0386-11ef-bc93-c7b262605968.lock 2024-04-26T04:33:37.779 INFO:teuthology.orchestra.run.smithi156.stdout:Lock 140481464967376 acquired on /run/cephadm/110027f2-0386-11ef-bc93-c7b262605968.lock 2024-04-26T04:33:37.779 INFO:teuthology.orchestra.run.smithi156.stdout:Verifying IP 172.21.15.156 port 3300 ... 2024-04-26T04:33:37.780 INFO:teuthology.orchestra.run.smithi156.stdout:Verifying IP 172.21.15.156 port 6789 ... 2024-04-26T04:33:37.780 INFO:teuthology.orchestra.run.smithi156.stdout:Base mon IP(s) is [172.21.15.156:3300, 172.21.15.156:6789], mon addrv is [v2:172.21.15.156:3300,v1:172.21.15.156:6789] 2024-04-26T04:33:37.783 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.156 metric 100 2024-04-26T04:33:37.783 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.156 metric 100 2024-04-26T04:33:37.786 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-04-26T04:33:37.786 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-04-26T04:33:37.786 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-04-26T04:33:37.788 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-26T04:33:37.789 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-04-26T04:33:37.789 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-26T04:33:37.789 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-26T04:33:37.789 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout inet6 fe80::14c8:ae95:f8b:2efb/64 scope link noprefixroute 2024-04-26T04:33:37.789 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-26T04:33:37.789 INFO:teuthology.orchestra.run.smithi156.stdout:Mon IP `172.21.15.156` is in CIDR network `172.21.0.0/20` 2024-04-26T04:33:37.790 INFO:teuthology.orchestra.run.smithi156.stdout:Mon IP `172.21.15.156` is in CIDR network `172.21.0.0/20` 2024-04-26T04:33:37.790 INFO:teuthology.orchestra.run.smithi156.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-04-26T04:33:37.790 INFO:teuthology.orchestra.run.smithi156.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-26T04:33:37.790 INFO:teuthology.orchestra.run.smithi156.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:33:38.285 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stdout 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 2024-04-26T04:33:38.285 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:33:38.285 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stderr Getting image source signatures 2024-04-26T04:33:38.285 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stderr Copying blob sha256:0e2af62137028b073c6ccdb8d4d7e0ccccc475ce5f6e225cb983e90813813045 2024-04-26T04:33:38.285 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stderr Copying blob sha256:3d665f4c032644be9cae748af04bee69f2bfe96cdc3616dabe0ab17899633ab5 2024-04-26T04:33:38.285 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stderr Copying config sha256:0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 2024-04-26T04:33:38.285 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stderr Writing manifest to image destination 2024-04-26T04:33:39.235 INFO:teuthology.orchestra.run.smithi156.stdout:ceph: stdout ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable) 2024-04-26T04:33:39.235 INFO:teuthology.orchestra.run.smithi156.stdout:Ceph version: ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable) 2024-04-26T04:33:39.235 INFO:teuthology.orchestra.run.smithi156.stdout:Extracting ceph user uid/gid from container image... 2024-04-26T04:33:40.252 INFO:teuthology.orchestra.run.smithi156.stdout:stat: stdout 167 167 2024-04-26T04:33:40.252 INFO:teuthology.orchestra.run.smithi156.stdout:Creating initial keys... 2024-04-26T04:33:41.310 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph-authtool: stdout AQCkLitm7voSJRAAvFaRk+VJLtuDzNFp/AFWTg== 2024-04-26T04:33:42.334 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph-authtool: stdout AQClLitmttfmJhAAH3aMsCIuwE0GKKes19wVdA== 2024-04-26T04:33:43.381 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph-authtool: stdout AQCmLitm3tjWKBAACUrjk+w6RNsqNI6IvGWUZw== 2024-04-26T04:33:43.382 INFO:teuthology.orchestra.run.smithi156.stdout:Creating initial monmap... 2024-04-26T04:33:44.321 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-26T04:33:44.322 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-04-26T04:33:44.322 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:33:44.322 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-26T04:33:44.322 INFO:teuthology.orchestra.run.smithi156.stdout:monmaptool for a [v2:172.21.15.156:3300,v1:172.21.15.156:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-26T04:33:44.322 INFO:teuthology.orchestra.run.smithi156.stdout:setting min_mon_release = octopus 2024-04-26T04:33:44.322 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/monmaptool: set fsid to 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:33:44.322 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-26T04:33:44.322 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:33:44.322 INFO:teuthology.orchestra.run.smithi156.stdout:Creating mon... 2024-04-26T04:33:45.319 INFO:teuthology.orchestra.run.smithi156.stdout:create mon.a on 2024-04-26T04:33:45.531 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-04-26T04:33:45.666 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-26T04:33:45.811 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-110027f2-0386-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-110027f2-0386-11ef-bc93-c7b262605968.target. 2024-04-26T04:33:45.811 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-110027f2-0386-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-110027f2-0386-11ef-bc93-c7b262605968.target. 2024-04-26T04:33:45.978 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.a 2024-04-26T04:33:45.978 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Failed to reset failed state of unit ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.a.service: Unit ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.a.service not loaded. 2024-04-26T04:33:46.135 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-110027f2-0386-11ef-bc93-c7b262605968.target.wants/ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.a.service → /etc/systemd/system/ceph-110027f2-0386-11ef-bc93-c7b262605968@.service. 2024-04-26T04:33:46.641 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:46 smithi156 podman[27057]: 2024-04-26 04:33:46.602265511 +0000 UTC m=+0.345351503 container start b47cf4ad009b8126772ff5821250e599951d34dc6f90159acb8a21b0fa8dac7d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T04:33:46.736 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:33:46.737 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout disabled 2024-04-26T04:33:46.742 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:33:46.742 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout inactive 2024-04-26T04:33:46.742 INFO:teuthology.orchestra.run.smithi156.stdout:firewalld.service is not enabled 2024-04-26T04:33:46.742 INFO:teuthology.orchestra.run.smithi156.stdout:Not possible to enable service . firewalld.service is not available 2024-04-26T04:33:46.743 INFO:teuthology.orchestra.run.smithi156.stdout:Waiting for mon to start... 2024-04-26T04:33:46.743 INFO:teuthology.orchestra.run.smithi156.stdout:Waiting for mon... 2024-04-26T04:33:46.968 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:46 smithi156 ceph-mon[27096]: mkfs 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:33:46.968 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:46 smithi156 ceph-mon[27096]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-26T04:33:46.968 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:46 smithi156 bash[27057]: b47cf4ad009b8126772ff5821250e599951d34dc6f90159acb8a21b0fa8dac7d 2024-04-26T04:33:46.968 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:46 smithi156 systemd[1]: Started Ceph mon.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:33:47.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:47 smithi156 ceph-mon[27096]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-26T04:33:47.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:47 smithi156 ceph-mon[27096]: monmap e1: 1 mons at {a=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:33:47.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:47 smithi156 ceph-mon[27096]: fsmap 2024-04-26T04:33:47.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:47 smithi156 ceph-mon[27096]: osdmap e1: 0 total, 0 up, 0 in 2024-04-26T04:33:47.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:47 smithi156 ceph-mon[27096]: mgrmap e1: no daemons active 2024-04-26T04:33:47.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:47 smithi156 ceph-mon[27096]: from='client.? 172.21.15.156:0/104019935' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T04:33:47.857 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout cluster: 2024-04-26T04:33:47.859 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout id: 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:33:47.859 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-04-26T04:33:47.859 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:47.859 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout services: 2024-04-26T04:33:47.859 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.644195s) 2024-04-26T04:33:47.859 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-04-26T04:33:47.859 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-04-26T04:33:47.860 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:47.860 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout data: 2024-04-26T04:33:47.860 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-04-26T04:33:47.860 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-04-26T04:33:47.860 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-04-26T04:33:47.860 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout pgs: 2024-04-26T04:33:47.860 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:47.860 INFO:teuthology.orchestra.run.smithi156.stdout:mon is available 2024-04-26T04:33:47.860 INFO:teuthology.orchestra.run.smithi156.stdout:Assimilating anything we can from ceph.conf... 2024-04-26T04:33:48.948 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:48 smithi156 ceph-mon[27096]: from='client.? 172.21.15.156:0/3688841707' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-26T04:33:48.948 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:48 smithi156 ceph-mon[27096]: from='client.? 172.21.15.156:0/3688841707' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-26T04:33:48.951 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:48.951 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout [global] 2024-04-26T04:33:48.951 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout fsid = 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:33:48.952 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.156:3300,v1:172.21.15.156:6789] 2024-04-26T04:33:48.952 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-26T04:33:48.952 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-26T04:33:48.952 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-26T04:33:48.952 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-26T04:33:48.952 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:48.952 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-26T04:33:48.952 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-26T04:33:48.952 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:48.952 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout [osd] 2024-04-26T04:33:48.952 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-26T04:33:48.952 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-26T04:33:48.952 INFO:teuthology.orchestra.run.smithi156.stdout:Generating new minimal ceph.conf... 2024-04-26T04:33:49.979 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:49 smithi156 ceph-mon[27096]: from='client.? 172.21.15.156:0/2154682165' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:33:49.982 INFO:teuthology.orchestra.run.smithi156.stdout:Restarting the monitor... 2024-04-26T04:33:50.236 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:49 smithi156 systemd[1]: Stopping Ceph mon.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:33:50.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:50 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a[27073]: 2024-04-26T04:33:50.234+0000 7ff968f98700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:33:50.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:50 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a[27073]: 2024-04-26T04:33:50.234+0000 7ff968f98700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-26T04:33:50.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:50 smithi156 podman[27418]: 2024-04-26 04:33:50.419259843 +0000 UTC m=+0.280732764 container died b47cf4ad009b8126772ff5821250e599951d34dc6f90159acb8a21b0fa8dac7d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424) 2024-04-26T04:33:50.991 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:50 smithi156 podman[27418]: 2024-04-26 04:33:50.789928638 +0000 UTC m=+0.651401563 container cleanup b47cf4ad009b8126772ff5821250e599951d34dc6f90159acb8a21b0fa8dac7d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True) 2024-04-26T04:33:50.991 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:50 smithi156 bash[27418]: ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a 2024-04-26T04:33:51.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:50 smithi156 podman[27437]: 2024-04-26 04:33:50.990800402 +0000 UTC m=+0.573563691 container remove b47cf4ad009b8126772ff5821250e599951d34dc6f90159acb8a21b0fa8dac7d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-26T04:33:51.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-26T04:33:51.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 systemd[1]: Stopped Ceph mon.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:33:51.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 systemd[1]: Starting Ceph mon.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:33:51.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 podman[27532]: 2024-04-26 04:33:51.341661683 +0000 UTC m=+0.113725295 container create 3d899f93d0ef43f8f8e9bdc515dcd625816d215720310e33a0e6709fa445ba00 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a, RELEASE=quincy-c61472c, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:33:51.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 podman[27532]: 2024-04-26 04:33:51.24531796 +0000 UTC m=+0.017381573 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph:quincy 2024-04-26T04:33:51.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 podman[27532]: 2024-04-26 04:33:51.52742565 +0000 UTC m=+0.299489263 container init 3d899f93d0ef43f8f8e9bdc515dcd625816d215720310e33a0e6709fa445ba00 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5) 2024-04-26T04:33:51.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 podman[27532]: 2024-04-26 04:33:51.532770264 +0000 UTC m=+0.304833880 container start 3d899f93d0ef43f8f8e9bdc515dcd625816d215720310e33a0e6709fa445ba00 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a, RELEASE=quincy-c61472c, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:33:51.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:33:51.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable), process ceph-mon, pid 7 2024-04-26T04:33:51.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: pidfile_write: ignore empty --pid-file 2024-04-26T04:33:51.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: load: jerasure load: lrc 2024-04-26T04:33:51.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: RocksDB version: 6.15.5 2024-04-26T04:33:51.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-26T04:33:51.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Compile date Apr 24 2024 2024-04-26T04:33:51.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: DB SUMMARY 2024-04-26T04:33:51.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: DB Session ID: HF7SE6QBB1NEO83422W4 2024-04-26T04:33:51.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:33:51.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:33:51.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-26T04:33:51.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-26T04:33:51.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 84258 ; 2024-04-26T04:33:51.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:33:51.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:33:51.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:33:51.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:33:51.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.env: 0x55dddb9e2b00 2024-04-26T04:33:51.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.fs: Posix File System 2024-04-26T04:33:51.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.info_log: 0x55dddcb5cee0 2024-04-26T04:33:51.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:33:51.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.statistics: (nil) 2024-04-26T04:33:51.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.use_fsync: 0 2024-04-26T04:33:51.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:33:51.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:33:51.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:33:51.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:33:51.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:33:51.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:33:51.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:33:51.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:33:51.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:33:51.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:33:51.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:33:51.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.db_log_dir: 2024-04-26T04:33:51.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-26T04:33:51.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:33:51.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:33:51.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:33:51.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:33:51.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:33:51.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:33:51.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:33:51.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:33:51.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.write_buffer_manager: 0x55dddd8cc930 2024-04-26T04:33:51.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:33:51.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-26T04:33:51.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:33:51.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:33:51.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:33:51.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:33:51.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:33:51.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:33:51.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:33:51.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.unordered_write: 0 2024-04-26T04:33:51.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:33:51.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:33:51.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:33:51.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:33:51.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.row_cache: None 2024-04-26T04:33:51.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.wal_filter: None 2024-04-26T04:33:51.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:33:51.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:33:51.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.preserve_deletes: 0 2024-04-26T04:33:51.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:33:51.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:33:51.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:33:51.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:33:51.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:33:51.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:33:51.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:33:51.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:33:51.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:33:51.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:33:51.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:33:51.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:33:51.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:33:51.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:33:51.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:33:51.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:33:51.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:33:51.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:33:51.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:33:51.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:33:51.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:33:51.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:33:51.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:33:51.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:33:51.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_open_files: -1 2024-04-26T04:33:51.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:33:51.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:33:51.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:33:51.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:33:51.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:33:51.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Compression algorithms supported: 2024-04-26T04:33:51.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:33:51.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: kZSTD supported: 0 2024-04-26T04:33:51.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:33:51.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:33:51.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:33:51.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:33:51.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:33:51.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:33:51.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:33:51.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-26T04:33:51.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-26T04:33:51.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:33:51.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.merge_operator: 2024-04-26T04:33:51.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compaction_filter: None 2024-04-26T04:33:51.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:33:51.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:33:51.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:33:51.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:33:51.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55dddcbacf58) 2024-04-26T04:33:51.606 INFO:journalctl@ceph.mon.a.smithi156.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:33:51.606 INFO:journalctl@ceph.mon.a.smithi156.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:33:51.606 INFO:journalctl@ceph.mon.a.smithi156.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:33:51.606 INFO:journalctl@ceph.mon.a.smithi156.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:33:51.606 INFO:journalctl@ceph.mon.a.smithi156.stdout: index_type: 0 2024-04-26T04:33:51.606 INFO:journalctl@ceph.mon.a.smithi156.stdout: data_block_index_type: 0 2024-04-26T04:33:51.606 INFO:journalctl@ceph.mon.a.smithi156.stdout: index_shortening: 1 2024-04-26T04:33:51.606 INFO:journalctl@ceph.mon.a.smithi156.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:33:51.606 INFO:journalctl@ceph.mon.a.smithi156.stdout: hash_index_allow_collision: 1 2024-04-26T04:33:51.606 INFO:journalctl@ceph.mon.a.smithi156.stdout: checksum: 1 2024-04-26T04:33:51.606 INFO:journalctl@ceph.mon.a.smithi156.stdout: no_block_cache: 0 2024-04-26T04:33:51.606 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_cache: 0x55dddcbdf090 2024-04-26T04:33:51.606 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:33:51.606 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_cache_options: 2024-04-26T04:33:51.607 INFO:journalctl@ceph.mon.a.smithi156.stdout: capacity : 536870912 2024-04-26T04:33:51.607 INFO:journalctl@ceph.mon.a.smithi156.stdout: num_shard_bits : 4 2024-04-26T04:33:51.607 INFO:journalctl@ceph.mon.a.smithi156.stdout: strict_capacity_limit : 0 2024-04-26T04:33:51.607 INFO:journalctl@ceph.mon.a.smithi156.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:33:51.607 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_cache_compressed: (nil) 2024-04-26T04:33:51.607 INFO:journalctl@ceph.mon.a.smithi156.stdout: persistent_cache: (nil) 2024-04-26T04:33:51.607 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_size: 4096 2024-04-26T04:33:51.607 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_size_deviation: 10 2024-04-26T04:33:51.607 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_restart_interval: 16 2024-04-26T04:33:51.607 INFO:journalctl@ceph.mon.a.smithi156.stdout: index_block_restart_interval: 1 2024-04-26T04:33:51.607 INFO:journalctl@ceph.mon.a.smithi156.stdout: metadata_block_size: 4096 2024-04-26T04:33:51.607 INFO:journalctl@ceph.mon.a.smithi156.stdout: partition_filters: 0 2024-04-26T04:33:51.607 INFO:journalctl@ceph.mon.a.smithi156.stdout: use_delta_encoding: 1 2024-04-26T04:33:51.607 INFO:journalctl@ceph.mon.a.smithi156.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-26T04:33:51.607 INFO:journalctl@ceph.mon.a.smithi156.stdout: whole_key_filtering: 1 2024-04-26T04:33:51.608 INFO:journalctl@ceph.mon.a.smithi156.stdout: verify_compression: 0 2024-04-26T04:33:51.608 INFO:journalctl@ceph.mon.a.smithi156.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:33:51.608 INFO:journalctl@ceph.mon.a.smithi156.stdout: format_version: 4 2024-04-26T04:33:51.608 INFO:journalctl@ceph.mon.a.smithi156.stdout: enable_index_compression: 1 2024-04-26T04:33:51.608 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_align: 0 2024-04-26T04:33:51.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:33:51.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:33:51.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compression: NoCompression 2024-04-26T04:33:51.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:33:51.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:33:51.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:33:51.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.num_levels: 7 2024-04-26T04:33:51.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:33:51.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:33:51.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:33:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:33:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:33:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:33:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:33:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:33:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:33:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:33:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:33:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:33:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:33:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:33:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:33:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:33:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:33:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:33:51.610 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:33:51.610 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:33:51.610 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:33:51.610 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:33:51.610 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:33:51.610 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:33:51.610 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:33:51.610 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:33:51.610 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:33:51.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:33:51.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:33:51.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:33:51.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:33:51.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:33:51.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:33:51.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:33:51.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.arena_block_size: 4194304 2024-04-26T04:33:51.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:33:51.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:33:51.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-26T04:33:51.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:33:51.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:33:51.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:33:51.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:33:51.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:33:51.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:33:51.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:33:51.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:33:51.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:33:51.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:33:51.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:33:51.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:33:51.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:33:51.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:33:51.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:33:51.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:33:51.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:33:51.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:33:51.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:33:51.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:33:51.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:33:51.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:33:51.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:33:51.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.ttl: 2592000 2024-04-26T04:33:51.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:33:51.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:33:51.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:33:51.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:33:51.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:33:51.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:33:51.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:33:51.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:33:51.615 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-04-26T04:33:51.615 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-04-26T04:33:51.615 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106031559056, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-26T04:33:51.615 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-04-26T04:33:51.615 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-26T04:33:51.615 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106031560117, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80690, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78951, "index_size": 227, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73159, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714106031, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "11e49592-8486-4c7b-abce-1cd7cefe9242", "db_session_id": "HF7SE6QBB1NEO83422W4"}} 2024-04-26T04:33:51.615 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-04-26T04:33:51.615 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106031560566, "job": 1, "event": "recovery_finished"} 2024-04-26T04:33:51.615 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:33:51.615 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55dddcb73180 2024-04-26T04:33:51.615 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: DB pointer 0x55dddcc48000 2024-04-26T04:33:51.615 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-26T04:33:51.615 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-26T04:33:51.615 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** DB Stats ** 2024-04-26T04:33:51.615 INFO:journalctl@ceph.mon.a.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:33:51.616 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:33:51.616 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:33:51.616 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:33:51.616 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:33:51.616 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-26T04:33:51.616 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:33:51.616 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-04-26T04:33:51.616 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** Compaction Stats [default] ** 2024-04-26T04:33:51.616 INFO:journalctl@ceph.mon.a.smithi156.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:33:51.616 INFO:journalctl@ceph.mon.a.smithi156.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:33:51.616 INFO:journalctl@ceph.mon.a.smithi156.stdout: L0 2/0 80.56 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 106.6 0.00 0.00 1 0.001 0 0 2024-04-26T04:33:51.616 INFO:journalctl@ceph.mon.a.smithi156.stdout: Sum 2/0 80.56 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 106.6 0.00 0.00 1 0.001 0 0 2024-04-26T04:33:51.616 INFO:journalctl@ceph.mon.a.smithi156.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 106.6 0.00 0.00 1 0.001 0 0 2024-04-26T04:33:51.616 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-04-26T04:33:51.616 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** Compaction Stats [default] ** 2024-04-26T04:33:51.617 INFO:journalctl@ceph.mon.a.smithi156.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:33:51.617 INFO:journalctl@ceph.mon.a.smithi156.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:33:51.617 INFO:journalctl@ceph.mon.a.smithi156.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 106.6 0.00 0.00 1 0.001 0 0 2024-04-26T04:33:51.617 INFO:journalctl@ceph.mon.a.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:33:51.617 INFO:journalctl@ceph.mon.a.smithi156.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:33:51.617 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:33:51.617 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:33:51.617 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:33:51.617 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:33:51.617 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative compaction: 0.00 GB write, 19.01 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:33:51.617 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval compaction: 0.00 GB write, 19.01 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:33:51.617 INFO:journalctl@ceph.mon.a.smithi156.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:33:51.617 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-04-26T04:33:51.617 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:33:51.617 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-04-26T04:33:51.617 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** Compaction Stats [default] ** 2024-04-26T04:33:51.618 INFO:journalctl@ceph.mon.a.smithi156.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:33:51.618 INFO:journalctl@ceph.mon.a.smithi156.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:33:51.618 INFO:journalctl@ceph.mon.a.smithi156.stdout: L0 2/0 80.56 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 106.6 0.00 0.00 1 0.001 0 0 2024-04-26T04:33:51.618 INFO:journalctl@ceph.mon.a.smithi156.stdout: Sum 2/0 80.56 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 106.6 0.00 0.00 1 0.001 0 0 2024-04-26T04:33:51.618 INFO:journalctl@ceph.mon.a.smithi156.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-26T04:33:51.618 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-04-26T04:33:51.618 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** Compaction Stats [default] ** 2024-04-26T04:33:51.618 INFO:journalctl@ceph.mon.a.smithi156.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:33:51.618 INFO:journalctl@ceph.mon.a.smithi156.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:33:51.618 INFO:journalctl@ceph.mon.a.smithi156.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 106.6 0.00 0.00 1 0.001 0 0 2024-04-26T04:33:51.618 INFO:journalctl@ceph.mon.a.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:33:51.618 INFO:journalctl@ceph.mon.a.smithi156.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:33:51.618 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:33:51.618 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:33:51.619 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:33:51.619 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:33:51.619 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative compaction: 0.00 GB write, 18.87 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:33:51.619 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:33:51.619 INFO:journalctl@ceph.mon.a.smithi156.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:33:51.619 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-04-26T04:33:51.619 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:33:51.619 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: starting mon.a rank 0 at public addrs [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] at bind addrs [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:33:51.619 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: mon.a@-1(???) e1 preinit fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:33:51.619 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: mon.a@-1(???).mds e1 new map 2024-04-26T04:33:51.619 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: mon.a@-1(???).mds e1 print_map 2024-04-26T04:33:51.619 INFO:journalctl@ceph.mon.a.smithi156.stdout: e1 2024-04-26T04:33:51.619 INFO:journalctl@ceph.mon.a.smithi156.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:33:51.619 INFO:journalctl@ceph.mon.a.smithi156.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:33:51.619 INFO:journalctl@ceph.mon.a.smithi156.stdout: legacy client fscid: -1 2024-04-26T04:33:51.619 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-04-26T04:33:51.620 INFO:journalctl@ceph.mon.a.smithi156.stdout: No filesystems configured 2024-04-26T04:33:51.620 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-26T04:33:51.620 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:33:51.620 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:33:51.620 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:33:51.620 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-26T04:33:51.693 INFO:teuthology.orchestra.run.smithi156.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-04-26T04:33:51.878 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-26T04:33:51.878 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: monmap e1: 1 mons at {a=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:33:51.878 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: fsmap 2024-04-26T04:33:51.878 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: osdmap e1: 0 total, 0 up, 0 in 2024-04-26T04:33:51.878 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 ceph-mon[27570]: mgrmap e1: no daemons active 2024-04-26T04:33:51.878 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 bash[27532]: 3d899f93d0ef43f8f8e9bdc515dcd625816d215720310e33a0e6709fa445ba00 2024-04-26T04:33:51.879 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:51 smithi156 systemd[1]: Started Ceph mon.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:33:52.738 INFO:teuthology.orchestra.run.smithi156.stdout:Wrote config to /etc/ceph/ceph.conf 2024-04-26T04:33:52.738 INFO:teuthology.orchestra.run.smithi156.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:33:52.738 INFO:teuthology.orchestra.run.smithi156.stdout:Creating mgr... 2024-04-26T04:33:52.738 INFO:teuthology.orchestra.run.smithi156.stdout:Verifying port 9283 ... 2024-04-26T04:33:52.915 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.y 2024-04-26T04:33:52.915 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Failed to reset failed state of unit ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.y.service: Unit ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.y.service not loaded. 2024-04-26T04:33:53.058 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-110027f2-0386-11ef-bc93-c7b262605968.target.wants/ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-110027f2-0386-11ef-bc93-c7b262605968@.service. 2024-04-26T04:33:53.182 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:53 smithi156 systemd[1]: Starting Ceph mgr.y for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:33:53.513 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:53 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3347636344' entity='client.admin' 2024-04-26T04:33:53.513 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:53 smithi156 podman[27806]: 2024-04-26 04:33:53.285547685 +0000 UTC m=+0.110840811 container create b232ee6793f9c41d372c6cb52800846e66d9cf46871d614441f698cf5b8e2c4a (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-26T04:33:53.513 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:53 smithi156 podman[27806]: 2024-04-26 04:33:53.193141186 +0000 UTC m=+0.018434328 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph:quincy 2024-04-26T04:33:53.514 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:53 smithi156 podman[27806]: 2024-04-26 04:33:53.513007781 +0000 UTC m=+0.338300907 container init b232ee6793f9c41d372c6cb52800846e66d9cf46871d614441f698cf5b8e2c4a (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-26T04:33:53.644 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:33:53.644 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout disabled 2024-04-26T04:33:53.651 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:33:53.651 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout inactive 2024-04-26T04:33:53.651 INFO:teuthology.orchestra.run.smithi156.stdout:firewalld.service is not enabled 2024-04-26T04:33:53.651 INFO:teuthology.orchestra.run.smithi156.stdout:Not possible to enable service . firewalld.service is not available 2024-04-26T04:33:53.660 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:33:53.660 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout disabled 2024-04-26T04:33:53.667 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:33:53.667 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout inactive 2024-04-26T04:33:53.667 INFO:teuthology.orchestra.run.smithi156.stdout:firewalld.service is not enabled 2024-04-26T04:33:53.667 INFO:teuthology.orchestra.run.smithi156.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-26T04:33:53.667 INFO:teuthology.orchestra.run.smithi156.stdout:Waiting for mgr to start... 2024-04-26T04:33:53.667 INFO:teuthology.orchestra.run.smithi156.stdout:Waiting for mgr... 2024-04-26T04:33:53.790 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:53 smithi156 podman[27806]: 2024-04-26 04:33:53.518560957 +0000 UTC m=+0.343854084 container start b232ee6793f9c41d372c6cb52800846e66d9cf46871d614441f698cf5b8e2c4a (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, ceph=True, RELEASE=quincy-c61472c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-26T04:33:53.790 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:53 smithi156 bash[27806]: b232ee6793f9c41d372c6cb52800846e66d9cf46871d614441f698cf5b8e2c4a 2024-04-26T04:33:53.790 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:53 smithi156 systemd[1]: Started Ceph mgr.y for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:33:53.790 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:53 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:53.659+0000 7fa6444701c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:33:54.083 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:53 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:53.824+0000 7fa6444701c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:33:54.083 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:53 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:53.930+0000 7fa6444701c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:33:54.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:33:54 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1997565378' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T04:33:54.344 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:54.248+0000 7fa6444701c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:33:54.344 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:54.330+0000 7fa6444701c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:33:54.779 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:54.779 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:33:54.779 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "fsid": "110027f2-0386-11ef-bc93-c7b262605968", 2024-04-26T04:33:54.779 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "health": { 2024-04-26T04:33:54.779 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-26T04:33:54.779 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-26T04:33:54.779 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-26T04:33:54.779 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:54.780 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-26T04:33:54.780 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-26T04:33:54.780 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 0 2024-04-26T04:33:54.780 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:33:54.780 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-26T04:33:54.780 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "a" 2024-04-26T04:33:54.780 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:33:54.780 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-04-26T04:33:54.780 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-26T04:33:54.780 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:33:54.780 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-26T04:33:54.780 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-26T04:33:54.780 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:54.781 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-26T04:33:54.781 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:33:54.781 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-26T04:33:54.781 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-26T04:33:54.781 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-26T04:33:54.781 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-26T04:33:54.781 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-26T04:33:54.781 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-26T04:33:54.781 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:54.781 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-26T04:33:54.781 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-26T04:33:54.781 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-26T04:33:54.781 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-26T04:33:54.782 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-26T04:33:54.782 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-26T04:33:54.782 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-26T04:33:54.782 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-26T04:33:54.782 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-26T04:33:54.782 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:54.782 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-26T04:33:54.782 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:33:54.782 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-26T04:33:54.782 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-26T04:33:54.782 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:54.782 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-26T04:33:54.783 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-26T04:33:54.783 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-26T04:33:54.783 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-26T04:33:54.783 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-26T04:33:54.783 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-26T04:33:54.783 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "restful" 2024-04-26T04:33:54.783 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:33:54.783 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:33:54.783 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:54.783 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-26T04:33:54.783 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:33:54.783 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "modified": "2024-04-26T04:33:46.638900+0000", 2024-04-26T04:33:54.784 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:33:54.784 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:54.784 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-26T04:33:54.784 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:33:54.784 INFO:teuthology.orchestra.run.smithi156.stdout:mgr not available, waiting (1/15)... 2024-04-26T04:33:55.812 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:55 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:55.487+0000 7fa6444701c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:33:55.812 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:55 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:55.645+0000 7fa6444701c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:33:55.812 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:55 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:55.811+0000 7fa6444701c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:33:56.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:56 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:56.065+0000 7fa6444701c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:33:56.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:56 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:56.153+0000 7fa6444701c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:33:56.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:56 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:56.243+0000 7fa6444701c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:33:56.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:56 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:56.519+0000 7fa6444701c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:33:57.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:56 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:56.977+0000 7fa6444701c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:33:57.765 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:57 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:57.500+0000 7fa6444701c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:33:57.765 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:57 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:57.591+0000 7fa6444701c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:33:57.765 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:57 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:57.764+0000 7fa6444701c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:33:58.328 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:58 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:58.052+0000 7fa6444701c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:33:58.328 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:58 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:58.312+0000 7fa6444701c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:33:58.593 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:58 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:58.545+0000 7fa6444701c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:33:58.977 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:58 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:58.677+0000 7fa6444701c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:33:58.977 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:58 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:58.768+0000 7fa6444701c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:33:58.977 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:58 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:58.883+0000 7fa6444701c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:33:58.977 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:58 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:58.976+0000 7fa6444701c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:34:00.083 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:59 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:59.764+0000 7fa6444701c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:34:00.083 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:33:59 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:33:59.955+0000 7fa6444701c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:34:00.083 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:00 smithi156 ceph-mon[27570]: Activating manager daemon y 2024-04-26T04:34:00.083 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:00 smithi156 ceph-mon[27570]: mgrmap e2: y(active, starting, since 0.00293872s) 2024-04-26T04:34:00.083 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:00 smithi156 ceph-mon[27570]: from='mgr.14100 172.21.15.156:0/1961265589' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:34:00.084 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:00 smithi156 ceph-mon[27570]: from='mgr.14100 172.21.15.156:0/1961265589' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:34:00.084 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:00 smithi156 ceph-mon[27570]: from='mgr.14100 172.21.15.156:0/1961265589' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:34:00.084 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:00 smithi156 ceph-mon[27570]: from='mgr.14100 172.21.15.156:0/1961265589' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:34:00.084 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:00 smithi156 ceph-mon[27570]: from='mgr.14100 172.21.15.156:0/1961265589' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:34:00.084 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:00 smithi156 ceph-mon[27570]: Manager daemon y is now available 2024-04-26T04:34:00.084 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:00 smithi156 ceph-mon[27570]: from='mgr.14100 172.21.15.156:0/1961265589' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:34:00.084 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:00 smithi156 ceph-mon[27570]: from='mgr.14100 172.21.15.156:0/1961265589' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:34:00.084 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:00 smithi156 ceph-mon[27570]: from='mgr.14100 172.21.15.156:0/1961265589' entity='mgr.y' 2024-04-26T04:34:00.084 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:00 smithi156 ceph-mon[27570]: from='mgr.14100 172.21.15.156:0/1961265589' entity='mgr.y' 2024-04-26T04:34:00.084 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:00 smithi156 ceph-mon[27570]: from='mgr.14100 172.21.15.156:0/1961265589' entity='mgr.y' 2024-04-26T04:34:01.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:01 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/370864052' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T04:34:01.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:01 smithi156 ceph-mon[27570]: mgrmap e3: y(active, since 1.00681s) 2024-04-26T04:34:03.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:03 smithi156 ceph-mon[27570]: mgrmap e4: y(active, since 2s) 2024-04-26T04:34:06.975 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-04-26T04:34:06.976 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:34:06.977 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "fsid": "110027f2-0386-11ef-bc93-c7b262605968", 2024-04-26T04:34:06.977 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "health": { 2024-04-26T04:34:06.977 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-26T04:34:06.977 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-26T04:34:06.977 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-26T04:34:06.977 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:34:06.977 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-26T04:34:06.978 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-26T04:34:06.978 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 0 2024-04-26T04:34:06.978 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:34:06.978 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-26T04:34:06.978 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "a" 2024-04-26T04:34:06.978 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:34:06.979 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum_age": 8, 2024-04-26T04:34:06.979 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-26T04:34:06.979 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:34:06.979 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-26T04:34:06.979 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-26T04:34:06.980 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:34:06.980 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-26T04:34:06.980 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:34:06.980 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-26T04:34:06.980 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-26T04:34:06.981 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-26T04:34:06.981 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-26T04:34:06.981 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-26T04:34:06.981 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-26T04:34:06.981 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:34:06.981 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-26T04:34:06.982 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-26T04:34:06.982 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-26T04:34:06.982 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-26T04:34:06.983 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-26T04:34:06.983 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-26T04:34:06.983 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-26T04:34:06.983 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-26T04:34:06.984 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-26T04:34:06.984 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:34:06.984 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-26T04:34:06.984 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:34:06.984 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-26T04:34:06.984 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-26T04:34:06.985 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:34:06.986 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-26T04:34:06.986 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-26T04:34:06.986 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-26T04:34:06.986 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-26T04:34:06.986 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-26T04:34:06.987 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-26T04:34:06.987 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "restful" 2024-04-26T04:34:06.987 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:34:06.987 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:34:06.987 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:34:06.987 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-26T04:34:06.988 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:34:06.988 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "modified": "2024-04-26T04:33:46.638900+0000", 2024-04-26T04:34:06.988 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:34:06.988 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:34:06.988 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-26T04:34:06.989 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:34:06.989 INFO:teuthology.orchestra.run.smithi156.stdout:mgr not available, waiting (2/15)... 2024-04-26T04:34:10.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:09 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3111733023' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T04:34:10.216 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-04-26T04:34:10.217 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:34:10.217 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "fsid": "110027f2-0386-11ef-bc93-c7b262605968", 2024-04-26T04:34:10.217 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "health": { 2024-04-26T04:34:10.217 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-26T04:34:10.217 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-26T04:34:10.217 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-26T04:34:10.217 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:34:10.217 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-26T04:34:10.217 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-26T04:34:10.217 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 0 2024-04-26T04:34:10.217 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:34:10.217 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-26T04:34:10.218 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "a" 2024-04-26T04:34:10.218 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:34:10.218 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum_age": 18, 2024-04-26T04:34:10.218 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-26T04:34:10.218 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:34:10.218 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-26T04:34:10.218 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-26T04:34:10.218 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:34:10.218 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-26T04:34:10.218 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:34:10.219 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-26T04:34:10.219 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-26T04:34:10.219 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-26T04:34:10.219 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-26T04:34:10.223 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-26T04:34:10.223 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-26T04:34:10.223 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:34:10.224 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-26T04:34:10.224 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-26T04:34:10.224 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-26T04:34:10.224 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-26T04:34:10.224 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-26T04:34:10.224 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-26T04:34:10.224 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-26T04:34:10.224 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-26T04:34:10.224 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-26T04:34:10.224 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:34:10.224 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-26T04:34:10.224 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:34:10.224 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-26T04:34:10.224 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-26T04:34:10.225 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:34:10.225 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-26T04:34:10.225 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-26T04:34:10.225 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-26T04:34:10.225 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-26T04:34:10.225 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-26T04:34:10.225 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-26T04:34:10.225 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "restful" 2024-04-26T04:34:10.225 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:34:10.225 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:34:10.225 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:34:10.225 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-26T04:34:10.226 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:34:10.226 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "modified": "2024-04-26T04:33:46.638900+0000", 2024-04-26T04:34:10.226 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:34:10.226 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:34:10.226 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-26T04:34:10.226 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:34:10.226 INFO:teuthology.orchestra.run.smithi156.stdout:mgr is available 2024-04-26T04:34:11.159 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:10 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/984851433' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-26T04:34:11.439 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-04-26T04:34:11.439 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout [global] 2024-04-26T04:34:11.439 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout fsid = 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:34:11.439 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-26T04:34:11.439 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-26T04:34:11.439 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-26T04:34:11.439 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-26T04:34:11.439 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-04-26T04:34:11.439 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-26T04:34:11.439 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-26T04:34:11.440 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-04-26T04:34:11.440 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout [osd] 2024-04-26T04:34:11.440 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-26T04:34:11.440 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-26T04:34:11.440 INFO:teuthology.orchestra.run.smithi156.stdout:Enabling cephadm module... 2024-04-26T04:34:12.525 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:12 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/221447217' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-26T04:34:12.525 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:12 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ignoring --setuser ceph since I am not root 2024-04-26T04:34:12.525 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:12 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ignoring --setgroup ceph since I am not root 2024-04-26T04:34:12.525 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:12 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:12.386+0000 7f286c6051c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:34:12.809 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:12 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:12.548+0000 7f286c6051c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:34:12.809 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:12 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:12.647+0000 7f286c6051c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:34:13.078 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:12 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:12.956+0000 7f286c6051c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:34:13.078 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:13 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:13.039+0000 7f286c6051c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:34:13.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:13 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/221447217' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-26T04:34:13.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:13 smithi156 ceph-mon[27570]: mgrmap e5: y(active, since 12s) 2024-04-26T04:34:14.169 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:34:14.169 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-04-26T04:34:14.169 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-26T04:34:14.169 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-26T04:34:14.169 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-26T04:34:14.169 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:34:14.169 INFO:teuthology.orchestra.run.smithi156.stdout:Waiting for the mgr to restart... 2024-04-26T04:34:14.169 INFO:teuthology.orchestra.run.smithi156.stdout:Waiting for mgr epoch 5... 2024-04-26T04:34:14.421 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:14 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/190666846' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-26T04:34:14.422 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:14.223+0000 7f286c6051c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:34:14.422 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:14.391+0000 7f286c6051c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:34:14.831 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:14.566+0000 7f286c6051c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:34:14.831 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:14.829+0000 7f286c6051c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:34:15.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:14.918+0000 7f286c6051c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:34:15.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:15.010+0000 7f286c6051c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:34:15.593 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:15.284+0000 7f286c6051c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:34:16.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:15.737+0000 7f286c6051c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:34:16.527 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:16.259+0000 7f286c6051c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:34:16.527 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:16.350+0000 7f286c6051c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:34:16.527 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:16.525+0000 7f286c6051c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:34:17.073 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:16.813+0000 7f286c6051c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:34:17.073 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:17.070+0000 7f286c6051c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:34:17.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:17.306+0000 7f286c6051c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:34:17.739 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:17.438+0000 7f286c6051c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:34:17.739 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:17.529+0000 7f286c6051c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:34:17.739 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:17.642+0000 7f286c6051c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:34:18.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:17.743+0000 7f286c6051c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:34:18.593 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:18.546+0000 7f286c6051c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:34:19.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:18.737+0000 7f286c6051c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:34:19.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-mon[27570]: Active manager daemon y restarted 2024-04-26T04:34:19.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-mon[27570]: Activating manager daemon y 2024-04-26T04:34:19.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-mon[27570]: osdmap e2: 0 total, 0 up, 0 in 2024-04-26T04:34:19.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-mon[27570]: mgrmap e6: y(active, starting, since 0.00307629s) 2024-04-26T04:34:19.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:34:19.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:34:19.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:34:19.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:34:19.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:34:19.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-mon[27570]: Manager daemon y is now available 2024-04-26T04:34:19.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' 2024-04-26T04:34:19.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' 2024-04-26T04:34:19.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:19.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:19.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:19.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:34:19.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:18 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:34:20.055 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:19 smithi156 ceph-mon[27570]: Found migration_current of "None". Setting to last migration. 2024-04-26T04:34:20.055 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:19 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' 2024-04-26T04:34:20.055 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:19 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:20.055 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:19 smithi156 ceph-mon[27570]: mgrmap e7: y(active, since 1.00475s) 2024-04-26T04:34:20.292 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:34:20.292 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-04-26T04:34:20.292 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-26T04:34:20.292 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:34:20.292 INFO:teuthology.orchestra.run.smithi156.stdout:mgr epoch 5 is available 2024-04-26T04:34:20.292 INFO:teuthology.orchestra.run.smithi156.stdout:Setting orchestrator backend to cephadm... 2024-04-26T04:34:20.978 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:20 smithi156 ceph-mon[27570]: [26/Apr/2024:04:34:19] ENGINE Bus STARTING 2024-04-26T04:34:20.978 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:20 smithi156 ceph-mon[27570]: [26/Apr/2024:04:34:19] ENGINE Serving on https://172.21.15.156:7150 2024-04-26T04:34:20.979 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:20 smithi156 ceph-mon[27570]: [26/Apr/2024:04:34:19] ENGINE Bus STARTED 2024-04-26T04:34:20.979 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:20 smithi156 ceph-mon[27570]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-26T04:34:20.979 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:20 smithi156 ceph-mon[27570]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-26T04:34:22.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:21 smithi156 ceph-mon[27570]: mgrmap e8: y(active, since 2s) 2024-04-26T04:34:22.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:21 smithi156 ceph-mon[27570]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:22.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:21 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' 2024-04-26T04:34:22.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:21 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:22.922 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout value unchanged 2024-04-26T04:34:22.922 INFO:teuthology.orchestra.run.smithi156.stdout:Generating ssh key... 2024-04-26T04:34:23.028 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:22 smithi156 ceph-mon[27570]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:23.983 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: Generating public/private rsa key pair. 2024-04-26T04:34:23.983 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: Your identification has been saved in /tmp/tmpddlvkyx4/key. 2024-04-26T04:34:23.983 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: Your public key has been saved in /tmp/tmpddlvkyx4/key.pub. 2024-04-26T04:34:23.984 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: The key fingerprint is: 2024-04-26T04:34:23.984 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: SHA256:epr5eaOb1QJI++LC/PUhtKq2Uuz/agzx7xoV6RcWZM0 ceph-110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:34:23.984 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: The key's randomart image is: 2024-04-26T04:34:23.984 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: +---[RSA 3072]----+ 2024-04-26T04:34:23.984 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: | .+o | 2024-04-26T04:34:23.984 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: | o .E | 2024-04-26T04:34:23.984 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: | . o o | 2024-04-26T04:34:23.984 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: | .. + o . | 2024-04-26T04:34:23.984 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: | . oo S . | 2024-04-26T04:34:23.984 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: | + .= + . | 2024-04-26T04:34:23.984 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: | = o+.* + . | 2024-04-26T04:34:23.984 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: | . *.oX.*oo | 2024-04-26T04:34:23.984 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: | ooB@*O+.. | 2024-04-26T04:34:23.984 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: +----[SHA256]-----+ 2024-04-26T04:34:24.910 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:24 smithi156 ceph-mon[27570]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:24.911 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:24 smithi156 ceph-mon[27570]: Generating ssh key... 2024-04-26T04:34:24.911 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:24 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' 2024-04-26T04:34:24.911 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:24 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' 2024-04-26T04:34:25.454 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAXKE0GWihyW2yPhAIXX4zIyP34sh2LlwB6lXQqf2D0bMfbVgS1teJFNLDzXmTj1IMPkQC7KWbgXRUYqDGY+IOrlVJbZYtggQb8B770zgDd4pVBn/yEE6EQYbwklW30uDQ9O2REnAjwWQdkBA4tZ26iW58NTiw48osrTbc84vjZgLvQkStce2woeAFLOZhw2dRQPAU5jjcHZqWdssfjGCUy3sYQKZUW4eyNRIrL1SXoBj2R6/X/eF8Bg+lM4HdyOFPepIAbOGH3qxvLEbmQ+Xdu+3tnDFDsDw+jHiZ+651t/1k3zgRNpbi57yVMHdGv2t1Wxrsm33siySzinYM09MAe3urbnMihGB4G+/tbaUD65V+IftNtuo1oSNZEsjlr2eAjNqEg9NIpfPFyUrKr98CIZz+kkjxSDOj+f5hz/5zXNLVTwpIxZ6YQ7qCtdV52dMTAsAd3gIERVuU267j5u6npHi5ApWXysbAY2+kTf4dKpktMDmWRZRHvRV0mj3ckwc= ceph-110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:34:25.454 INFO:teuthology.orchestra.run.smithi156.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-26T04:34:25.454 INFO:teuthology.orchestra.run.smithi156.stdout:Adding key to root@localhost authorized_keys... 2024-04-26T04:34:25.455 INFO:teuthology.orchestra.run.smithi156.stdout:Adding host smithi156... 2024-04-26T04:34:25.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:25 smithi156 ceph-mon[27570]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:26.934 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:26 smithi156 ceph-mon[27570]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi156", "addr": "172.21.15.156", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:28.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:27 smithi156 ceph-mon[27570]: Deploying cephadm binary to smithi156 2024-04-26T04:34:28.476 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout Added host 'smithi156' with addr '172.21.15.156' 2024-04-26T04:34:28.476 INFO:teuthology.orchestra.run.smithi156.stdout:Deploying unmanaged mon service... 2024-04-26T04:34:29.206 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:28 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' 2024-04-26T04:34:29.206 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:28 smithi156 ceph-mon[27570]: Added host smithi156 2024-04-26T04:34:29.206 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:28 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:30.574 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:30 smithi156 ceph-mon[27570]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:30.575 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:30 smithi156 ceph-mon[27570]: Saving service mon spec with placement count:5 2024-04-26T04:34:30.575 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:30 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' 2024-04-26T04:34:30.792 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-04-26T04:34:30.792 INFO:teuthology.orchestra.run.smithi156.stdout:Deploying unmanaged mgr service... 2024-04-26T04:34:31.516 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:31 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' 2024-04-26T04:34:31.516 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:31 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' 2024-04-26T04:34:32.091 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-04-26T04:34:32.753 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:32 smithi156 ceph-mon[27570]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:32.753 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:32 smithi156 ceph-mon[27570]: Saving service mgr spec with placement count:2 2024-04-26T04:34:32.753 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:32 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' 2024-04-26T04:34:33.773 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:33 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/4190013499' entity='client.admin' 2024-04-26T04:34:33.774 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:33 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' 2024-04-26T04:34:35.042 INFO:teuthology.orchestra.run.smithi156.stdout:Enabling the dashboard module... 2024-04-26T04:34:35.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:35 smithi156 ceph-mon[27570]: from='mgr.14118 172.21.15.156:0/3506148779' entity='mgr.y' 2024-04-26T04:34:35.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:35 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2811568366' entity='client.admin' 2024-04-26T04:34:36.532 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:36 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/4219761166' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-26T04:34:36.533 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:36 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ignoring --setuser ceph since I am not root 2024-04-26T04:34:36.533 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:36 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ignoring --setgroup ceph since I am not root 2024-04-26T04:34:36.533 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:36 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:36.474+0000 7fec43f1a1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:34:36.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:36 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:36.636+0000 7fec43f1a1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:34:36.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:36 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:36.735+0000 7fec43f1a1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:34:37.128 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:37 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:37.044+0000 7fec43f1a1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:34:37.128 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:37 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:37.126+0000 7fec43f1a1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:34:37.396 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:37 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/4219761166' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-26T04:34:37.396 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:37 smithi156 ceph-mon[27570]: mgrmap e9: y(active, since 17s) 2024-04-26T04:34:38.482 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:38 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/239160660' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-26T04:34:38.482 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:38 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:38.317+0000 7fec43f1a1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:34:38.483 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:38 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:38.481+0000 7fec43f1a1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:34:38.505 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:34:38.506 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-04-26T04:34:38.506 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-26T04:34:38.506 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-26T04:34:38.506 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-26T04:34:38.506 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:34:38.506 INFO:teuthology.orchestra.run.smithi156.stdout:Waiting for the mgr to restart... 2024-04-26T04:34:38.506 INFO:teuthology.orchestra.run.smithi156.stdout:Waiting for mgr epoch 9... 2024-04-26T04:34:38.770 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:38 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:38.667+0000 7fec43f1a1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:34:39.022 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:38 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:38.944+0000 7fec43f1a1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:34:39.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:39 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:39.039+0000 7fec43f1a1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:34:39.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:39 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:39.137+0000 7fec43f1a1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:34:39.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:39 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:39.425+0000 7fec43f1a1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:34:40.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:39 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:39.891+0000 7fec43f1a1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:34:40.691 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:40 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:40.421+0000 7fec43f1a1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:34:40.691 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:40 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:40.513+0000 7fec43f1a1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:34:40.691 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:40 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:40.690+0000 7fec43f1a1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:34:41.240 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:40 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:40.979+0000 7fec43f1a1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:34:41.240 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:41 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:41.239+0000 7fec43f1a1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:34:41.593 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:41 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:41.477+0000 7fec43f1a1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:34:41.906 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:41 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:41.609+0000 7fec43f1a1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:34:41.907 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:41 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:41.701+0000 7fec43f1a1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:34:41.907 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:41 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:41.816+0000 7fec43f1a1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:34:41.907 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:41 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:41.905+0000 7fec43f1a1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:34:42.995 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:42.713+0000 7fec43f1a1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:34:42.996 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:34:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:34:42.906+0000 7fec43f1a1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:34:42.996 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:42 smithi156 ceph-mon[27570]: Active manager daemon y restarted 2024-04-26T04:34:42.996 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:42 smithi156 ceph-mon[27570]: Activating manager daemon y 2024-04-26T04:34:42.996 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:42 smithi156 ceph-mon[27570]: osdmap e3: 0 total, 0 up, 0 in 2024-04-26T04:34:42.996 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:42 smithi156 ceph-mon[27570]: mgrmap e10: y(active, starting, since 0.00312238s) 2024-04-26T04:34:42.996 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:34:42.996 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:34:42.996 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:34:42.997 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:34:42.997 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:34:42.997 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:42 smithi156 ceph-mon[27570]: Manager daemon y is now available 2024-04-26T04:34:42.997 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:42.997 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:42.997 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:34:42.997 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:34:44.480 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:34:44.481 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-04-26T04:34:44.481 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-26T04:34:44.481 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:34:44.481 INFO:teuthology.orchestra.run.smithi156.stdout:mgr epoch 9 is available 2024-04-26T04:34:44.481 INFO:teuthology.orchestra.run.smithi156.stdout:Generating a dashboard self-signed certificate... 2024-04-26T04:34:45.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:44 smithi156 ceph-mon[27570]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-26T04:34:45.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:44 smithi156 ceph-mon[27570]: mgrmap e11: y(active, since 1.00546s) 2024-04-26T04:34:45.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:44 smithi156 ceph-mon[27570]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-26T04:34:45.846 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-04-26T04:34:45.846 INFO:teuthology.orchestra.run.smithi156.stdout:Creating initial admin user... 2024-04-26T04:34:46.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:45 smithi156 ceph-mon[27570]: mgrmap e12: y(active, since 2s) 2024-04-26T04:34:46.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:45 smithi156 ceph-mon[27570]: [26/Apr/2024:04:34:44] ENGINE Bus STARTING 2024-04-26T04:34:46.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:45 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:34:46.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:45 smithi156 ceph-mon[27570]: [26/Apr/2024:04:34:45] ENGINE Serving on https://172.21.15.156:7150 2024-04-26T04:34:46.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:45 smithi156 ceph-mon[27570]: [26/Apr/2024:04:34:45] ENGINE Bus STARTED 2024-04-26T04:34:46.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:45 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:34:46.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:45 smithi156 ceph-mon[27570]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:46.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:45 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:34:46.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:45 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:34:47.727 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:47 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:34:47.727 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:47 smithi156 ceph-mon[27570]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:47.728 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:47 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:34:48.067 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$hwB8BtMZ0cuGHZbTBqHJWuCcf0z6BXOyPz0BuGWEXnJuSqnthJXK2", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1714106086, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-26T04:34:48.067 INFO:teuthology.orchestra.run.smithi156.stdout:Fetching dashboard port number... 2024-04-26T04:34:49.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:48 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3742851299' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-26T04:34:49.331 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 8443 2024-04-26T04:34:49.340 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:34:49.340 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout disabled 2024-04-26T04:34:49.347 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:34:49.348 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout inactive 2024-04-26T04:34:49.348 INFO:teuthology.orchestra.run.smithi156.stdout:firewalld.service is not enabled 2024-04-26T04:34:49.348 INFO:teuthology.orchestra.run.smithi156.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-26T04:34:49.350 INFO:teuthology.orchestra.run.smithi156.stdout:Ceph Dashboard is now available at: 2024-04-26T04:34:49.350 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:34:49.350 INFO:teuthology.orchestra.run.smithi156.stdout: URL: https://smithi156:8443/ 2024-04-26T04:34:49.350 INFO:teuthology.orchestra.run.smithi156.stdout: User: admin 2024-04-26T04:34:49.350 INFO:teuthology.orchestra.run.smithi156.stdout: Password: 3ls7zpnlnf 2024-04-26T04:34:49.350 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:34:49.350 INFO:teuthology.orchestra.run.smithi156.stdout:Saving cluster configuration to /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config directory 2024-04-26T04:34:49.350 INFO:teuthology.orchestra.run.smithi156.stdout:Enabling autotune for osd_memory_target 2024-04-26T04:34:51.336 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:50 smithi156 ceph-mon[27570]: mgrmap e13: y(active, since 6s) 2024-04-26T04:34:51.877 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-04-26T04:34:51.877 INFO:teuthology.orchestra.run.smithi156.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-04-26T04:34:51.877 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:34:51.877 INFO:teuthology.orchestra.run.smithi156.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:34:51.877 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:34:51.877 INFO:teuthology.orchestra.run.smithi156.stdout:Or, if you are only running a single cluster on this host: 2024-04-26T04:34:51.878 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:34:51.878 INFO:teuthology.orchestra.run.smithi156.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-04-26T04:34:51.878 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:34:51.878 INFO:teuthology.orchestra.run.smithi156.stdout:Please consider enabling telemetry to help improve Ceph: 2024-04-26T04:34:51.878 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:34:51.878 INFO:teuthology.orchestra.run.smithi156.stdout: ceph telemetry on 2024-04-26T04:34:51.878 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:34:51.878 INFO:teuthology.orchestra.run.smithi156.stdout:For more information see: 2024-04-26T04:34:51.878 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:34:51.878 INFO:teuthology.orchestra.run.smithi156.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-26T04:34:51.878 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:34:51.878 INFO:teuthology.orchestra.run.smithi156.stdout:Bootstrap complete. 2024-04-26T04:34:51.913 INFO:tasks.cephadm:Fetching config... 2024-04-26T04:34:51.913 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-04-26T04:34:51.914 DEBUG:teuthology.orchestra.run.smithi156:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-26T04:34:51.931 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-26T04:34:51.931 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-04-26T04:34:51.931 DEBUG:teuthology.orchestra.run.smithi156:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-26T04:34:51.987 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-26T04:34:51.987 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-04-26T04:34:51.987 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd if=/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/keyring of=/dev/stdout 2024-04-26T04:34:52.051 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-26T04:34:52.051 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-04-26T04:34:52.051 DEBUG:teuthology.orchestra.run.smithi156:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-26T04:34:52.105 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-26T04:34:52.106 DEBUG:teuthology.orchestra.run.smithi156:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAXKE0GWihyW2yPhAIXX4zIyP34sh2LlwB6lXQqf2D0bMfbVgS1teJFNLDzXmTj1IMPkQC7KWbgXRUYqDGY+IOrlVJbZYtggQb8B770zgDd4pVBn/yEE6EQYbwklW30uDQ9O2REnAjwWQdkBA4tZ26iW58NTiw48osrTbc84vjZgLvQkStce2woeAFLOZhw2dRQPAU5jjcHZqWdssfjGCUy3sYQKZUW4eyNRIrL1SXoBj2R6/X/eF8Bg+lM4HdyOFPepIAbOGH3qxvLEbmQ+Xdu+3tnDFDsDw+jHiZ+651t/1k3zgRNpbi57yVMHdGv2t1Wxrsm33siySzinYM09MAe3urbnMihGB4G+/tbaUD65V+IftNtuo1oSNZEsjlr2eAjNqEg9NIpfPFyUrKr98CIZz+kkjxSDOj+f5hz/5zXNLVTwpIxZ6YQ7qCtdV52dMTAsAd3gIERVuU267j5u6npHi5ApWXysbAY2+kTf4dKpktMDmWRZRHvRV0mj3ckwc= ceph-110027f2-0386-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-26T04:34:52.177 INFO:teuthology.orchestra.run.smithi156.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAXKE0GWihyW2yPhAIXX4zIyP34sh2LlwB6lXQqf2D0bMfbVgS1teJFNLDzXmTj1IMPkQC7KWbgXRUYqDGY+IOrlVJbZYtggQb8B770zgDd4pVBn/yEE6EQYbwklW30uDQ9O2REnAjwWQdkBA4tZ26iW58NTiw48osrTbc84vjZgLvQkStce2woeAFLOZhw2dRQPAU5jjcHZqWdssfjGCUy3sYQKZUW4eyNRIrL1SXoBj2R6/X/eF8Bg+lM4HdyOFPepIAbOGH3qxvLEbmQ+Xdu+3tnDFDsDw+jHiZ+651t/1k3zgRNpbi57yVMHdGv2t1Wxrsm33siySzinYM09MAe3urbnMihGB4G+/tbaUD65V+IftNtuo1oSNZEsjlr2eAjNqEg9NIpfPFyUrKr98CIZz+kkjxSDOj+f5hz/5zXNLVTwpIxZ6YQ7qCtdV52dMTAsAd3gIERVuU267j5u6npHi5ApWXysbAY2+kTf4dKpktMDmWRZRHvRV0mj3ckwc= ceph-110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:34:52.189 DEBUG:teuthology.orchestra.run.smithi171:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAXKE0GWihyW2yPhAIXX4zIyP34sh2LlwB6lXQqf2D0bMfbVgS1teJFNLDzXmTj1IMPkQC7KWbgXRUYqDGY+IOrlVJbZYtggQb8B770zgDd4pVBn/yEE6EQYbwklW30uDQ9O2REnAjwWQdkBA4tZ26iW58NTiw48osrTbc84vjZgLvQkStce2woeAFLOZhw2dRQPAU5jjcHZqWdssfjGCUy3sYQKZUW4eyNRIrL1SXoBj2R6/X/eF8Bg+lM4HdyOFPepIAbOGH3qxvLEbmQ+Xdu+3tnDFDsDw+jHiZ+651t/1k3zgRNpbi57yVMHdGv2t1Wxrsm33siySzinYM09MAe3urbnMihGB4G+/tbaUD65V+IftNtuo1oSNZEsjlr2eAjNqEg9NIpfPFyUrKr98CIZz+kkjxSDOj+f5hz/5zXNLVTwpIxZ6YQ7qCtdV52dMTAsAd3gIERVuU267j5u6npHi5ApWXysbAY2+kTf4dKpktMDmWRZRHvRV0mj3ckwc= ceph-110027f2-0386-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-26T04:34:52.228 INFO:teuthology.orchestra.run.smithi171.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAXKE0GWihyW2yPhAIXX4zIyP34sh2LlwB6lXQqf2D0bMfbVgS1teJFNLDzXmTj1IMPkQC7KWbgXRUYqDGY+IOrlVJbZYtggQb8B770zgDd4pVBn/yEE6EQYbwklW30uDQ9O2REnAjwWQdkBA4tZ26iW58NTiw48osrTbc84vjZgLvQkStce2woeAFLOZhw2dRQPAU5jjcHZqWdssfjGCUy3sYQKZUW4eyNRIrL1SXoBj2R6/X/eF8Bg+lM4HdyOFPepIAbOGH3qxvLEbmQ+Xdu+3tnDFDsDw+jHiZ+651t/1k3zgRNpbi57yVMHdGv2t1Wxrsm33siySzinYM09MAe3urbnMihGB4G+/tbaUD65V+IftNtuo1oSNZEsjlr2eAjNqEg9NIpfPFyUrKr98CIZz+kkjxSDOj+f5hz/5zXNLVTwpIxZ6YQ7qCtdV52dMTAsAd3gIERVuU267j5u6npHi5ApWXysbAY2+kTf4dKpktMDmWRZRHvRV0mj3ckwc= ceph-110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:34:52.240 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-26T04:34:52.444 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:52 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2279315982' entity='client.admin' 2024-04-26T04:34:52.509 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:34:54.919 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:54 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:34:54.919 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:54 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:34:54.919 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:54 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi156", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:34:54.919 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:54 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:34:54.919 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:54 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:54.919 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:54 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:34:56.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:56 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1836932670' entity='client.admin' 2024-04-26T04:34:56.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:56 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:56.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:56 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:34:56.423 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-26T04:34:56.423 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-26T04:34:56.677 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:34:59.158 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi171 2024-04-26T04:34:59.158 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-04-26T04:34:59.158 DEBUG:teuthology.orchestra.run.smithi171:> dd of=/etc/ceph/ceph.conf 2024-04-26T04:34:59.176 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-04-26T04:34:59.176 DEBUG:teuthology.orchestra.run.smithi171:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:34:59.234 INFO:tasks.cephadm:Adding host smithi171 to orchestrator... 2024-04-26T04:34:59.234 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph orch host add smithi171 2024-04-26T04:34:59.485 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:34:59.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:59 smithi156 ceph-mon[27570]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:59.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:34:59.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:59.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:59.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:34:59.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:59 smithi156 ceph-mon[27570]: Updating smithi156:/etc/ceph/ceph.conf 2024-04-26T04:34:59.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:59 smithi156 ceph-mon[27570]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:34:59.712 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:59 smithi156 ceph-mon[27570]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:34:59.712 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:59 smithi156 ceph-mon[27570]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:34:59.712 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:34:59.712 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:34:59.712 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:34:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:01.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:01 smithi156 ceph-mon[27570]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi171", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:35:02.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:02 smithi156 ceph-mon[27570]: Deploying cephadm binary to smithi171 2024-04-26T04:35:03.141 INFO:teuthology.orchestra.run.smithi156.stdout:Added host 'smithi171' with addr '172.21.15.171' 2024-04-26T04:35:03.744 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-26T04:35:04.000 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:35:04.449 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:04 smithi156 ceph-mon[27570]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:04.449 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:04 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:04.449 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:04 smithi156 ceph-mon[27570]: Added host smithi171 2024-04-26T04:35:04.449 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:04 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:04.449 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:04 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:04.449 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:04 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:05.704 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:35:05.704 INFO:teuthology.orchestra.run.smithi156.stdout:[{"addr": "172.21.15.156", "hostname": "smithi156", "labels": [], "status": ""}, {"addr": "172.21.15.171", "hostname": "smithi171", "labels": [], "status": ""}] 2024-04-26T04:35:05.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:05 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:05.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:05 smithi156 ceph-mon[27570]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:06.273 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-26T04:35:06.273 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-26T04:35:06.517 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:35:06.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:06 smithi156 ceph-mon[27570]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:35:07.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:07 smithi156 ceph-mon[27570]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:07.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:07 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:07.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:07 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:07.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:07 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:07.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:07 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:07.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:07 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi171", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:35:07.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:07 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:07.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:07 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:09.292 INFO:teuthology.orchestra.run.smithi156.stderr:adjusted tunables profile to default 2024-04-26T04:35:09.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:09 smithi156 ceph-mon[27570]: Updating smithi171:/etc/ceph/ceph.conf 2024-04-26T04:35:09.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:09 smithi156 ceph-mon[27570]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:35:09.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:09 smithi156 ceph-mon[27570]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:35:09.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:09 smithi156 ceph-mon[27570]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:35:09.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:09 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:09.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:09 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:09.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:09 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:09.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:09 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3796733918' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-26T04:35:09.852 INFO:tasks.cephadm:Adding mon.a on smithi156 2024-04-26T04:35:09.853 INFO:tasks.cephadm:Adding mon.c on smithi156 2024-04-26T04:35:09.853 INFO:tasks.cephadm:Adding mon.b on smithi171 2024-04-26T04:35:09.853 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph orch apply mon '3;smithi156:172.21.15.156=a;smithi156:[v2:172.21.15.156:3301,v1:172.21.15.156:6790]=c;smithi171:172.21.15.171=b' 2024-04-26T04:35:10.102 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:35:10.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:10 smithi156 ceph-mon[27570]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:10.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:10 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3796733918' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-26T04:35:10.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:10 smithi156 ceph-mon[27570]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:35:11.859 INFO:teuthology.orchestra.run.smithi171.stdout:Scheduled mon update... 2024-04-26T04:35:12.497 DEBUG:teuthology.orchestra.run.smithi156:mon.c> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.c.service 2024-04-26T04:35:12.499 DEBUG:teuthology.orchestra.run.smithi171:mon.b> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.b.service 2024-04-26T04:35:12.501 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T04:35:12.502 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T04:35:12.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:12 smithi156 ceph-mon[27570]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:12.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:12 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:12.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:12 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:12.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:12 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:12.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:12 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:12.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:12 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:12.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:12 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:35:12.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:12 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:13.157 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:35:13.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:13 smithi156 ceph-mon[27570]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi156:172.21.15.156=a;smithi156:[v2:172.21.15.156:3301,v1:172.21.15.156:6790]=c;smithi171:172.21.15.171=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:35:13.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:13 smithi156 ceph-mon[27570]: Saving service mon spec with placement smithi156:172.21.15.156=a;smithi156:[v2:172.21.15.156:3301,v1:172.21.15.156:6790]=c;smithi171:172.21.15.171=b;count:3 2024-04-26T04:35:13.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:13 smithi156 ceph-mon[27570]: Deploying daemon mon.b on smithi171 2024-04-26T04:35:14.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:14 smithi156 ceph-mon[27570]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:16.310 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 podman[31073]: 2024-04-26 04:35:16.197433817 +0000 UTC m=+0.323257917 container create fb359e8b680cea83c4e9f9b6a566897361752fcd27c19510a16d0c8e0ec7aeec (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=quincy-c61472c, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-26T04:35:16.396 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:35:16.397 INFO:teuthology.orchestra.run.smithi171.stdout:{"epoch":1,"fsid":"110027f2-0386-11ef-bc93-c7b262605968","modified":"2024-04-26T04:33:43.779990Z","created":"2024-04-26T04:33:43.779990Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:3300","nonce":0},{"type":"v1","addr":"172.21.15.156:6789","nonce":0}]},"addr":"172.21.15.156:6789/0","public_addr":"172.21.15.156:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-26T04:35:16.399 INFO:teuthology.orchestra.run.smithi171.stderr:dumped monmap epoch 1 2024-04-26T04:35:16.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:16 smithi156 ceph-mon[27570]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:16.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 podman[31073]: 2024-04-26 04:35:16.392110739 +0000 UTC m=+0.517934840 container init fb359e8b680cea83c4e9f9b6a566897361752fcd27c19510a16d0c8e0ec7aeec (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, ceph=True) 2024-04-26T04:35:16.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 podman[31073]: 2024-04-26 04:35:16.39782212 +0000 UTC m=+0.523646221 container start fb359e8b680cea83c4e9f9b6a566897361752fcd27c19510a16d0c8e0ec7aeec (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-b, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-26T04:35:16.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:35:16.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable), process ceph-mon, pid 7 2024-04-26T04:35:16.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: pidfile_write: ignore empty --pid-file 2024-04-26T04:35:16.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: load: jerasure load: lrc 2024-04-26T04:35:16.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: RocksDB version: 6.15.5 2024-04-26T04:35:16.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-26T04:35:16.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Compile date Apr 24 2024 2024-04-26T04:35:16.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: DB SUMMARY 2024-04-26T04:35:16.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: DB Session ID: C3XWZAMLF738E0T6YWZ3 2024-04-26T04:35:16.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:35:16.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:35:16.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-26T04:35:16.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-04-26T04:35:16.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-04-26T04:35:16.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:35:16.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:35:16.752 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:35:16.752 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:35:16.752 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.env: 0x55dd69cd5b00 2024-04-26T04:35:16.752 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.fs: Posix File System 2024-04-26T04:35:16.752 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.info_log: 0x55dd6c26cee0 2024-04-26T04:35:16.752 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:35:16.752 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.statistics: (nil) 2024-04-26T04:35:16.752 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.use_fsync: 0 2024-04-26T04:35:16.752 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:35:16.752 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:35:16.752 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:35:16.752 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:35:16.752 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:35:16.752 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:35:16.753 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:35:16.753 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:35:16.753 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:35:16.753 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:35:16.753 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:35:16.753 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.db_log_dir: 2024-04-26T04:35:16.753 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-04-26T04:35:16.753 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:35:16.753 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:35:16.753 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:35:16.753 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:35:16.753 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:35:16.753 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:35:16.753 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:35:16.753 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:35:16.754 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.write_buffer_manager: 0x55dd6cfdc930 2024-04-26T04:35:16.754 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:35:16.754 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-26T04:35:16.754 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:35:16.754 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:35:16.754 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:35:16.754 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:35:16.754 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:35:16.754 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:35:16.754 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:35:16.754 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.unordered_write: 0 2024-04-26T04:35:16.754 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:35:16.754 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:35:16.754 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:35:16.755 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:35:16.755 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.row_cache: None 2024-04-26T04:35:16.755 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.wal_filter: None 2024-04-26T04:35:16.755 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:35:16.756 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:35:16.756 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.preserve_deletes: 0 2024-04-26T04:35:16.756 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:35:16.756 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:35:16.756 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:35:16.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:35:16.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:35:16.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:35:16.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:35:16.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:35:16.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:35:16.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:35:16.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:35:16.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:35:16.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:35:16.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:35:16.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:35:16.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:35:16.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:35:16.758 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:35:16.758 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:35:16.758 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:35:16.758 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:35:16.758 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:35:16.758 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:35:16.758 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:35:16.758 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_open_files: -1 2024-04-26T04:35:16.758 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:35:16.758 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:35:16.758 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:35:16.758 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:35:16.758 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:35:16.758 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Compression algorithms supported: 2024-04-26T04:35:16.759 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:35:16.759 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: kZSTD supported: 0 2024-04-26T04:35:16.759 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:35:16.759 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:35:16.759 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:35:16.759 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:35:16.759 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:35:16.759 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:35:16.759 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:35:16.759 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-04-26T04:35:16.759 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-26T04:35:16.759 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:35:16.759 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.merge_operator: 2024-04-26T04:35:16.759 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compaction_filter: None 2024-04-26T04:35:16.759 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:35:16.760 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:35:16.760 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:35:16.760 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:35:16.760 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55dd6c2bcf58) 2024-04-26T04:35:16.760 INFO:journalctl@ceph.mon.b.smithi171.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:35:16.760 INFO:journalctl@ceph.mon.b.smithi171.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:35:16.760 INFO:journalctl@ceph.mon.b.smithi171.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:35:16.760 INFO:journalctl@ceph.mon.b.smithi171.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:35:16.760 INFO:journalctl@ceph.mon.b.smithi171.stdout: index_type: 0 2024-04-26T04:35:16.760 INFO:journalctl@ceph.mon.b.smithi171.stdout: data_block_index_type: 0 2024-04-26T04:35:16.760 INFO:journalctl@ceph.mon.b.smithi171.stdout: index_shortening: 1 2024-04-26T04:35:16.760 INFO:journalctl@ceph.mon.b.smithi171.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:35:16.760 INFO:journalctl@ceph.mon.b.smithi171.stdout: hash_index_allow_collision: 1 2024-04-26T04:35:16.760 INFO:journalctl@ceph.mon.b.smithi171.stdout: checksum: 1 2024-04-26T04:35:16.760 INFO:journalctl@ceph.mon.b.smithi171.stdout: no_block_cache: 0 2024-04-26T04:35:16.761 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_cache: 0x55dd6c2ef090 2024-04-26T04:35:16.761 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:35:16.761 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_cache_options: 2024-04-26T04:35:16.761 INFO:journalctl@ceph.mon.b.smithi171.stdout: capacity : 536870912 2024-04-26T04:35:16.761 INFO:journalctl@ceph.mon.b.smithi171.stdout: num_shard_bits : 4 2024-04-26T04:35:16.761 INFO:journalctl@ceph.mon.b.smithi171.stdout: strict_capacity_limit : 0 2024-04-26T04:35:16.761 INFO:journalctl@ceph.mon.b.smithi171.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:35:16.761 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_cache_compressed: (nil) 2024-04-26T04:35:16.761 INFO:journalctl@ceph.mon.b.smithi171.stdout: persistent_cache: (nil) 2024-04-26T04:35:16.761 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_size: 4096 2024-04-26T04:35:16.761 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_size_deviation: 10 2024-04-26T04:35:16.761 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_restart_interval: 16 2024-04-26T04:35:16.761 INFO:journalctl@ceph.mon.b.smithi171.stdout: index_block_restart_interval: 1 2024-04-26T04:35:16.761 INFO:journalctl@ceph.mon.b.smithi171.stdout: metadata_block_size: 4096 2024-04-26T04:35:16.761 INFO:journalctl@ceph.mon.b.smithi171.stdout: partition_filters: 0 2024-04-26T04:35:16.762 INFO:journalctl@ceph.mon.b.smithi171.stdout: use_delta_encoding: 1 2024-04-26T04:35:16.762 INFO:journalctl@ceph.mon.b.smithi171.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-26T04:35:16.762 INFO:journalctl@ceph.mon.b.smithi171.stdout: whole_key_filtering: 1 2024-04-26T04:35:16.762 INFO:journalctl@ceph.mon.b.smithi171.stdout: verify_compression: 0 2024-04-26T04:35:16.762 INFO:journalctl@ceph.mon.b.smithi171.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:35:16.762 INFO:journalctl@ceph.mon.b.smithi171.stdout: format_version: 4 2024-04-26T04:35:16.762 INFO:journalctl@ceph.mon.b.smithi171.stdout: enable_index_compression: 1 2024-04-26T04:35:16.762 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_align: 0 2024-04-26T04:35:16.762 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:35:16.762 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:35:16.762 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compression: NoCompression 2024-04-26T04:35:16.762 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:35:16.762 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:35:16.763 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:35:16.763 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.num_levels: 7 2024-04-26T04:35:16.763 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:35:16.763 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:35:16.763 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:35:16.763 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:35:16.763 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:35:16.763 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:35:16.763 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:35:16.763 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:35:16.763 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:35:16.763 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:35:16.763 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:35:16.763 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:35:16.763 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:35:16.764 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:35:16.764 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:35:16.764 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:35:16.764 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:35:16.764 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:35:16.764 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:35:16.764 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:35:16.764 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:35:16.764 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:35:16.764 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:35:16.764 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:35:16.764 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:35:16.764 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:35:16.764 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:35:16.765 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:35:16.765 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:35:16.765 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:35:16.765 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:35:16.765 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:35:16.765 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:35:16.765 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:35:16.765 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.arena_block_size: 4194304 2024-04-26T04:35:16.765 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:35:16.765 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:35:16.765 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-26T04:35:16.765 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:35:16.765 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:35:16.765 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:35:16.765 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:35:16.766 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:35:16.766 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:35:16.766 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:35:16.766 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:35:16.766 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:35:16.766 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:35:16.766 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:35:16.766 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:35:16.766 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:35:16.766 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:35:16.766 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:35:16.766 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:35:16.766 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:35:16.766 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:35:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:35:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:35:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:35:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:35:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:35:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.ttl: 2592000 2024-04-26T04:35:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:35:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:35:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:35:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:35:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:35:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:35:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:35:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:35:16.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-04-26T04:35:16.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-04-26T04:35:16.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106116427112, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-26T04:35:16.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-04-26T04:35:16.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-26T04:35:16.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106116427443, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714106116, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "8ccd95c9-732a-43bb-be23-80f8cb0299ba", "db_session_id": "C3XWZAMLF738E0T6YWZ3"}} 2024-04-26T04:35:16.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-04-26T04:35:16.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106116427839, "job": 1, "event": "recovery_finished"} 2024-04-26T04:35:16.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:35:16.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55dd6c283180 2024-04-26T04:35:16.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: DB pointer 0x55dd6c358000 2024-04-26T04:35:16.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-26T04:35:16.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-26T04:35:16.769 INFO:journalctl@ceph.mon.b.smithi171.stdout: ** DB Stats ** 2024-04-26T04:35:16.769 INFO:journalctl@ceph.mon.b.smithi171.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:35:16.769 INFO:journalctl@ceph.mon.b.smithi171.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:35:16.769 INFO:journalctl@ceph.mon.b.smithi171.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:35:16.769 INFO:journalctl@ceph.mon.b.smithi171.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:35:16.769 INFO:journalctl@ceph.mon.b.smithi171.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:35:16.769 INFO:journalctl@ceph.mon.b.smithi171.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-26T04:35:16.769 INFO:journalctl@ceph.mon.b.smithi171.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:35:16.769 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-04-26T04:35:16.769 INFO:journalctl@ceph.mon.b.smithi171.stdout: ** Compaction Stats [default] ** 2024-04-26T04:35:16.769 INFO:journalctl@ceph.mon.b.smithi171.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:35:16.769 INFO:journalctl@ceph.mon.b.smithi171.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:35:16.769 INFO:journalctl@ceph.mon.b.smithi171.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:35:16.769 INFO:journalctl@ceph.mon.b.smithi171.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:35:16.769 INFO:journalctl@ceph.mon.b.smithi171.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:35:16.770 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-04-26T04:35:16.770 INFO:journalctl@ceph.mon.b.smithi171.stdout: ** Compaction Stats [default] ** 2024-04-26T04:35:16.770 INFO:journalctl@ceph.mon.b.smithi171.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:35:16.770 INFO:journalctl@ceph.mon.b.smithi171.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:35:16.770 INFO:journalctl@ceph.mon.b.smithi171.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:35:16.770 INFO:journalctl@ceph.mon.b.smithi171.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:35:16.770 INFO:journalctl@ceph.mon.b.smithi171.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:35:16.770 INFO:journalctl@ceph.mon.b.smithi171.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:35:16.770 INFO:journalctl@ceph.mon.b.smithi171.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:35:16.770 INFO:journalctl@ceph.mon.b.smithi171.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:35:16.770 INFO:journalctl@ceph.mon.b.smithi171.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:35:16.770 INFO:journalctl@ceph.mon.b.smithi171.stdout: Cumulative compaction: 0.00 GB write, 0.50 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:35:16.770 INFO:journalctl@ceph.mon.b.smithi171.stdout: Interval compaction: 0.00 GB write, 0.50 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:35:16.770 INFO:journalctl@ceph.mon.b.smithi171.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:35:16.771 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-04-26T04:35:16.771 INFO:journalctl@ceph.mon.b.smithi171.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:35:16.771 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-04-26T04:35:16.771 INFO:journalctl@ceph.mon.b.smithi171.stdout: ** Compaction Stats [default] ** 2024-04-26T04:35:16.771 INFO:journalctl@ceph.mon.b.smithi171.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:35:16.771 INFO:journalctl@ceph.mon.b.smithi171.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:35:16.771 INFO:journalctl@ceph.mon.b.smithi171.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:35:16.771 INFO:journalctl@ceph.mon.b.smithi171.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:35:16.771 INFO:journalctl@ceph.mon.b.smithi171.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-26T04:35:16.771 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-04-26T04:35:16.771 INFO:journalctl@ceph.mon.b.smithi171.stdout: ** Compaction Stats [default] ** 2024-04-26T04:35:16.771 INFO:journalctl@ceph.mon.b.smithi171.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:35:16.771 INFO:journalctl@ceph.mon.b.smithi171.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:35:16.772 INFO:journalctl@ceph.mon.b.smithi171.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-04-26T04:35:16.772 INFO:journalctl@ceph.mon.b.smithi171.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:35:16.772 INFO:journalctl@ceph.mon.b.smithi171.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:35:16.772 INFO:journalctl@ceph.mon.b.smithi171.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:35:16.772 INFO:journalctl@ceph.mon.b.smithi171.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:35:16.772 INFO:journalctl@ceph.mon.b.smithi171.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:35:16.772 INFO:journalctl@ceph.mon.b.smithi171.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:35:16.772 INFO:journalctl@ceph.mon.b.smithi171.stdout: Cumulative compaction: 0.00 GB write, 0.49 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:35:16.772 INFO:journalctl@ceph.mon.b.smithi171.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:35:16.772 INFO:journalctl@ceph.mon.b.smithi171.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:35:16.772 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-04-26T04:35:16.772 INFO:journalctl@ceph.mon.b.smithi171.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:35:16.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-04-26T04:35:16.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: using public_addr v2:172.21.15.171:0/0 -> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] 2024-04-26T04:35:16.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: starting mon.b rank -1 at public addrs [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] at bind addrs [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:35:16.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: mon.b@-1(???) e0 preinit fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:35:16.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: mon.b@-1(synchronizing).mds e1 new map 2024-04-26T04:35:16.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: mon.b@-1(synchronizing).mds e1 print_map 2024-04-26T04:35:16.773 INFO:journalctl@ceph.mon.b.smithi171.stdout: e1 2024-04-26T04:35:16.773 INFO:journalctl@ceph.mon.b.smithi171.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:35:16.773 INFO:journalctl@ceph.mon.b.smithi171.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:35:16.773 INFO:journalctl@ceph.mon.b.smithi171.stdout: legacy client fscid: -1 2024-04-26T04:35:16.773 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-04-26T04:35:16.773 INFO:journalctl@ceph.mon.b.smithi171.stdout: No filesystems configured 2024-04-26T04:35:16.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-26T04:35:16.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-26T04:35:16.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-26T04:35:16.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-26T04:35:16.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-26T04:35:16.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-26T04:35:16.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-26T04:35:16.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:35:16.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:35:16.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:35:16.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:16.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3796733918' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-26T04:35:16.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:35:16.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:16.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:16.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:16.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:16.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:16.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:16.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:35:16.775 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:16.775 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi156:172.21.15.156=a;smithi156:[v2:172.21.15.156:3301,v1:172.21.15.156:6790]=c;smithi171:172.21.15.171=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:35:16.775 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: Saving service mon spec with placement smithi156:172.21.15.156=a;smithi156:[v2:172.21.15.156:3301,v1:172.21.15.156:6790]=c;smithi171:172.21.15.171=b;count:3 2024-04-26T04:35:16.775 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: Deploying daemon mon.b on smithi171 2024-04-26T04:35:16.775 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:16.775 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:16.775 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 ceph-mon[31139]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-26T04:35:16.775 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 bash[31073]: fb359e8b680cea83c4e9f9b6a566897361752fcd27c19510a16d0c8e0ec7aeec 2024-04-26T04:35:16.775 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:16 smithi171 systemd[1]: Started Ceph mon.b for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:35:18.121 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T04:35:18.122 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T04:35:18.378 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:35:21.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:35:21.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: mon.a calling monitor election 2024-04-26T04:35:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: mon.b calling monitor election 2024-04-26T04:35:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:21.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:21.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T04:35:21.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: monmap e2: 2 mons at {a=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],b=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:35:21.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: fsmap 2024-04-26T04:35:21.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:35:21.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: mgrmap e13: y(active, since 38s) 2024-04-26T04:35:21.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: overall HEALTH_OK 2024-04-26T04:35:21.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:21.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:21.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:21.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:35:21.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:22.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:35:22.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: mon.a calling monitor election 2024-04-26T04:35:22.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:22.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:22.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:22.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:22.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: mon.b calling monitor election 2024-04-26T04:35:22.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:22.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:22.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:22.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:22.001 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:22.001 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T04:35:22.001 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: monmap e2: 2 mons at {a=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],b=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:35:22.001 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: fsmap 2024-04-26T04:35:22.001 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:35:22.001 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: mgrmap e13: y(active, since 38s) 2024-04-26T04:35:22.001 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: overall HEALTH_OK 2024-04-26T04:35:22.001 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:22.001 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:22.001 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:22.001 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:35:22.001 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:22.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:22 smithi156 ceph-mon[27570]: Deploying daemon mon.c on smithi156 2024-04-26T04:35:22.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:22 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:22.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:22 smithi171 ceph-mon[31139]: Deploying daemon mon.c on smithi156 2024-04-26T04:35:22.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:22 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:23.096 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:35:23.096 INFO:teuthology.orchestra.run.smithi171.stdout:{"epoch":2,"fsid":"110027f2-0386-11ef-bc93-c7b262605968","modified":"2024-04-26T04:35:16.451729Z","created":"2024-04-26T04:33:43.779990Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:3300","nonce":0},{"type":"v1","addr":"172.21.15.156:6789","nonce":0}]},"addr":"172.21.15.156:6789/0","public_addr":"172.21.15.156:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:3300","nonce":0},{"type":"v1","addr":"172.21.15.171:6789","nonce":0}]},"addr":"172.21.15.171:6789/0","public_addr":"172.21.15.171:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-26T04:35:23.098 INFO:teuthology.orchestra.run.smithi171.stderr:dumped monmap epoch 2 2024-04-26T04:35:23.681 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:23 smithi156 ceph-mon[27570]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:23.681 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:23 smithi156 ceph-mon[27570]: from='client.? 172.21.15.171:0/3909522661' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:35:23.688 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:23 smithi171 ceph-mon[31139]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:23.688 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:23 smithi171 ceph-mon[31139]: from='client.? 172.21.15.171:0/3909522661' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:35:24.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 podman[39092]: 2024-04-26 04:35:24.549655649 +0000 UTC m=+0.134312249 container create fc34a850791e9377c80d935c903d844b5d0ee2fe0ef2d6d2cab230f7534c9c0e (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-26T04:35:24.722 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T04:35:24.722 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T04:35:24.971 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:35:25.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 podman[39092]: 2024-04-26 04:35:24.718834955 +0000 UTC m=+0.303491552 container init fc34a850791e9377c80d935c903d844b5d0ee2fe0ef2d6d2cab230f7534c9c0e (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-26T04:35:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 podman[39092]: 2024-04-26 04:35:24.723977949 +0000 UTC m=+0.308634546 container start fc34a850791e9377c80d935c903d844b5d0ee2fe0ef2d6d2cab230f7534c9c0e (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-26T04:35:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:35:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable), process ceph-mon, pid 7 2024-04-26T04:35:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: pidfile_write: ignore empty --pid-file 2024-04-26T04:35:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: load: jerasure load: lrc 2024-04-26T04:35:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: RocksDB version: 6.15.5 2024-04-26T04:35:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-26T04:35:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Compile date Apr 24 2024 2024-04-26T04:35:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: DB SUMMARY 2024-04-26T04:35:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: DB Session ID: OWIEUL3T1M1KL300AC01 2024-04-26T04:35:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:35:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:35:25.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-26T04:35:25.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-04-26T04:35:25.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-04-26T04:35:25.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:35:25.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:35:25.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:35:25.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:35:25.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.env: 0x55edf4cd2b00 2024-04-26T04:35:25.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.fs: Posix File System 2024-04-26T04:35:25.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.info_log: 0x55edf6b60ee0 2024-04-26T04:35:25.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:35:25.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.statistics: (nil) 2024-04-26T04:35:25.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.use_fsync: 0 2024-04-26T04:35:25.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:35:25.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:35:25.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:35:25.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:35:25.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:35:25.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:35:25.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:35:25.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:35:25.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:35:25.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:35:25.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:35:25.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.db_log_dir: 2024-04-26T04:35:25.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-04-26T04:35:25.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:35:25.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:35:25.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:35:25.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:35:25.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:35:25.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:35:25.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:35:25.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:35:25.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.write_buffer_manager: 0x55edf78d0960 2024-04-26T04:35:25.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:35:25.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-26T04:35:25.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:35:25.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:35:25.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:35:25.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:35:25.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:35:25.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:35:25.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:35:25.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.unordered_write: 0 2024-04-26T04:35:25.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:35:25.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:35:25.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:35:25.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:35:25.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.row_cache: None 2024-04-26T04:35:25.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.wal_filter: None 2024-04-26T04:35:25.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:35:25.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:35:25.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.preserve_deletes: 0 2024-04-26T04:35:25.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:35:25.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:35:25.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:35:25.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:35:25.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:35:25.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:35:25.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:35:25.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:35:25.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:35:25.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:35:25.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:35:25.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:35:25.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:35:25.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:35:25.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:35:25.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:35:25.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:35:25.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:35:25.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:35:25.101 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:35:25.101 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:35:25.101 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:35:25.101 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:35:25.101 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:35:25.101 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_open_files: -1 2024-04-26T04:35:25.101 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:35:25.101 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:35:25.101 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:35:25.101 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:35:25.101 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:35:25.101 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Compression algorithms supported: 2024-04-26T04:35:25.101 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:35:25.101 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: kZSTD supported: 0 2024-04-26T04:35:25.102 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:35:25.102 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:35:25.102 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:35:25.102 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:35:25.102 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:35:25.102 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:35:25.102 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:35:25.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-04-26T04:35:25.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-26T04:35:25.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:35:25.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.merge_operator: 2024-04-26T04:35:25.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compaction_filter: None 2024-04-26T04:35:25.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:35:25.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:35:25.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:35:25.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:35:25.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55edf6bb0f58) 2024-04-26T04:35:25.104 INFO:journalctl@ceph.mon.c.smithi156.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:35:25.104 INFO:journalctl@ceph.mon.c.smithi156.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:35:25.104 INFO:journalctl@ceph.mon.c.smithi156.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:35:25.104 INFO:journalctl@ceph.mon.c.smithi156.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:35:25.104 INFO:journalctl@ceph.mon.c.smithi156.stdout: index_type: 0 2024-04-26T04:35:25.104 INFO:journalctl@ceph.mon.c.smithi156.stdout: data_block_index_type: 0 2024-04-26T04:35:25.104 INFO:journalctl@ceph.mon.c.smithi156.stdout: index_shortening: 1 2024-04-26T04:35:25.105 INFO:journalctl@ceph.mon.c.smithi156.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:35:25.105 INFO:journalctl@ceph.mon.c.smithi156.stdout: hash_index_allow_collision: 1 2024-04-26T04:35:25.105 INFO:journalctl@ceph.mon.c.smithi156.stdout: checksum: 1 2024-04-26T04:35:25.105 INFO:journalctl@ceph.mon.c.smithi156.stdout: no_block_cache: 0 2024-04-26T04:35:25.105 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_cache: 0x55edf6be3090 2024-04-26T04:35:25.105 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:35:25.105 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_cache_options: 2024-04-26T04:35:25.105 INFO:journalctl@ceph.mon.c.smithi156.stdout: capacity : 536870912 2024-04-26T04:35:25.105 INFO:journalctl@ceph.mon.c.smithi156.stdout: num_shard_bits : 4 2024-04-26T04:35:25.105 INFO:journalctl@ceph.mon.c.smithi156.stdout: strict_capacity_limit : 0 2024-04-26T04:35:25.105 INFO:journalctl@ceph.mon.c.smithi156.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:35:25.105 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_cache_compressed: (nil) 2024-04-26T04:35:25.105 INFO:journalctl@ceph.mon.c.smithi156.stdout: persistent_cache: (nil) 2024-04-26T04:35:25.105 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_size: 4096 2024-04-26T04:35:25.105 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_size_deviation: 10 2024-04-26T04:35:25.106 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_restart_interval: 16 2024-04-26T04:35:25.106 INFO:journalctl@ceph.mon.c.smithi156.stdout: index_block_restart_interval: 1 2024-04-26T04:35:25.106 INFO:journalctl@ceph.mon.c.smithi156.stdout: metadata_block_size: 4096 2024-04-26T04:35:25.106 INFO:journalctl@ceph.mon.c.smithi156.stdout: partition_filters: 0 2024-04-26T04:35:25.106 INFO:journalctl@ceph.mon.c.smithi156.stdout: use_delta_encoding: 1 2024-04-26T04:35:25.106 INFO:journalctl@ceph.mon.c.smithi156.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-26T04:35:25.106 INFO:journalctl@ceph.mon.c.smithi156.stdout: whole_key_filtering: 1 2024-04-26T04:35:25.106 INFO:journalctl@ceph.mon.c.smithi156.stdout: verify_compression: 0 2024-04-26T04:35:25.106 INFO:journalctl@ceph.mon.c.smithi156.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:35:25.106 INFO:journalctl@ceph.mon.c.smithi156.stdout: format_version: 4 2024-04-26T04:35:25.106 INFO:journalctl@ceph.mon.c.smithi156.stdout: enable_index_compression: 1 2024-04-26T04:35:25.106 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_align: 0 2024-04-26T04:35:25.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:35:25.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:35:25.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compression: NoCompression 2024-04-26T04:35:25.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:35:25.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:35:25.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:35:25.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.num_levels: 7 2024-04-26T04:35:25.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:35:25.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:35:25.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:35:25.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:35:25.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:35:25.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:35:25.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:35:25.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:35:25.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:35:25.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:35:25.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:35:25.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:35:25.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:35:25.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:35:25.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:35:25.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:35:25.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:35:25.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:35:25.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:35:25.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:35:25.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:35:25.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:35:25.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:35:25.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:35:25.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:35:25.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:35:25.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:35:25.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:35:25.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:35:25.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:35:25.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:35:25.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:35:25.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:35:25.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:35:25.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.arena_block_size: 4194304 2024-04-26T04:35:25.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:35:25.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:35:25.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-26T04:35:25.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:35:25.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:35:25.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:35:25.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:35:25.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:35:25.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:35:25.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:35:25.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:35:25.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:35:25.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:35:25.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:35:25.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:35:25.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:35:25.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:35:25.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:35:25.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:35:25.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:35:25.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:35:25.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:35:25.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:35:25.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:35:25.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:35:25.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:35:25.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.ttl: 2592000 2024-04-26T04:35:25.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:35:25.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:35:25.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:35:25.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:35:25.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:35:25.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:35:25.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:35:25.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:35:25.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-04-26T04:35:25.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-04-26T04:35:25.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106124750578, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-26T04:35:25.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-04-26T04:35:25.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-26T04:35:25.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106124750904, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714106124, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "b64b9110-2f6d-4d32-a695-922085150e22", "db_session_id": "OWIEUL3T1M1KL300AC01"}} 2024-04-26T04:35:25.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-04-26T04:35:25.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106124751217, "job": 1, "event": "recovery_finished"} 2024-04-26T04:35:25.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:35:25.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55edf6b77180 2024-04-26T04:35:25.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: DB pointer 0x55edf6c4c000 2024-04-26T04:35:25.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-26T04:35:25.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-26T04:35:25.113 INFO:journalctl@ceph.mon.c.smithi156.stdout: ** DB Stats ** 2024-04-26T04:35:25.113 INFO:journalctl@ceph.mon.c.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:35:25.113 INFO:journalctl@ceph.mon.c.smithi156.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:35:25.113 INFO:journalctl@ceph.mon.c.smithi156.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:35:25.114 INFO:journalctl@ceph.mon.c.smithi156.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:35:25.114 INFO:journalctl@ceph.mon.c.smithi156.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:35:25.114 INFO:journalctl@ceph.mon.c.smithi156.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-26T04:35:25.114 INFO:journalctl@ceph.mon.c.smithi156.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:35:25.114 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-04-26T04:35:25.114 INFO:journalctl@ceph.mon.c.smithi156.stdout: ** Compaction Stats [default] ** 2024-04-26T04:35:25.114 INFO:journalctl@ceph.mon.c.smithi156.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:35:25.114 INFO:journalctl@ceph.mon.c.smithi156.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:35:25.114 INFO:journalctl@ceph.mon.c.smithi156.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:35:25.114 INFO:journalctl@ceph.mon.c.smithi156.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:35:25.114 INFO:journalctl@ceph.mon.c.smithi156.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:35:25.114 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-04-26T04:35:25.114 INFO:journalctl@ceph.mon.c.smithi156.stdout: ** Compaction Stats [default] ** 2024-04-26T04:35:25.115 INFO:journalctl@ceph.mon.c.smithi156.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:35:25.115 INFO:journalctl@ceph.mon.c.smithi156.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:35:25.115 INFO:journalctl@ceph.mon.c.smithi156.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:35:25.115 INFO:journalctl@ceph.mon.c.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:35:25.115 INFO:journalctl@ceph.mon.c.smithi156.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:35:25.115 INFO:journalctl@ceph.mon.c.smithi156.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:35:25.115 INFO:journalctl@ceph.mon.c.smithi156.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:35:25.115 INFO:journalctl@ceph.mon.c.smithi156.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:35:25.115 INFO:journalctl@ceph.mon.c.smithi156.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:35:25.115 INFO:journalctl@ceph.mon.c.smithi156.stdout: Cumulative compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:35:25.115 INFO:journalctl@ceph.mon.c.smithi156.stdout: Interval compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:35:25.115 INFO:journalctl@ceph.mon.c.smithi156.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:35:25.115 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-04-26T04:35:25.115 INFO:journalctl@ceph.mon.c.smithi156.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:35:25.116 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-04-26T04:35:25.116 INFO:journalctl@ceph.mon.c.smithi156.stdout: ** Compaction Stats [default] ** 2024-04-26T04:35:25.116 INFO:journalctl@ceph.mon.c.smithi156.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:35:25.116 INFO:journalctl@ceph.mon.c.smithi156.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:35:25.116 INFO:journalctl@ceph.mon.c.smithi156.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:35:25.116 INFO:journalctl@ceph.mon.c.smithi156.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:35:25.116 INFO:journalctl@ceph.mon.c.smithi156.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-26T04:35:25.116 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-04-26T04:35:25.116 INFO:journalctl@ceph.mon.c.smithi156.stdout: ** Compaction Stats [default] ** 2024-04-26T04:35:25.116 INFO:journalctl@ceph.mon.c.smithi156.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:35:25.116 INFO:journalctl@ceph.mon.c.smithi156.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:35:25.116 INFO:journalctl@ceph.mon.c.smithi156.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-04-26T04:35:25.116 INFO:journalctl@ceph.mon.c.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:35:25.116 INFO:journalctl@ceph.mon.c.smithi156.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:35:25.116 INFO:journalctl@ceph.mon.c.smithi156.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:35:25.117 INFO:journalctl@ceph.mon.c.smithi156.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:35:25.117 INFO:journalctl@ceph.mon.c.smithi156.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:35:25.117 INFO:journalctl@ceph.mon.c.smithi156.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:35:25.117 INFO:journalctl@ceph.mon.c.smithi156.stdout: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:35:25.117 INFO:journalctl@ceph.mon.c.smithi156.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:35:25.117 INFO:journalctl@ceph.mon.c.smithi156.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:35:25.117 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-04-26T04:35:25.117 INFO:journalctl@ceph.mon.c.smithi156.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:35:25.117 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-04-26T04:35:25.117 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: using public_addrv [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] 2024-04-26T04:35:25.118 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: starting mon.c rank -1 at public addrs [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] at bind addrs [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:35:25.118 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.c@-1(???) e0 preinit fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:35:25.118 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.c@-1(synchronizing).mds e1 new map 2024-04-26T04:35:25.118 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.c@-1(synchronizing).mds e1 print_map 2024-04-26T04:35:25.118 INFO:journalctl@ceph.mon.c.smithi156.stdout: e1 2024-04-26T04:35:25.118 INFO:journalctl@ceph.mon.c.smithi156.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:35:25.118 INFO:journalctl@ceph.mon.c.smithi156.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:35:25.118 INFO:journalctl@ceph.mon.c.smithi156.stdout: legacy client fscid: -1 2024-04-26T04:35:25.118 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-04-26T04:35:25.118 INFO:journalctl@ceph.mon.c.smithi156.stdout: No filesystems configured 2024-04-26T04:35:25.118 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-26T04:35:25.118 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-26T04:35:25.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-26T04:35:25.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-26T04:35:25.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-26T04:35:25.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-26T04:35:25.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-26T04:35:25.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:35:25.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:35:25.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:35:25.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:25.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3796733918' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-26T04:35:25.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:35:25.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:25.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:25.120 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:25.120 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:25.120 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:25.120 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:25.120 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:35:25.120 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:25.120 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi156:172.21.15.156=a;smithi156:[v2:172.21.15.156:3301,v1:172.21.15.156:6790]=c;smithi171:172.21.15.171=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:35:25.120 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: Saving service mon spec with placement smithi156:172.21.15.156=a;smithi156:[v2:172.21.15.156:3301,v1:172.21.15.156:6790]=c;smithi171:172.21.15.171=b;count:3 2024-04-26T04:35:25.120 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: Deploying daemon mon.b on smithi171 2024-04-26T04:35:25.120 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:25.120 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:25.120 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:35:25.120 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.a calling monitor election 2024-04-26T04:35:25.120 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:25.120 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:25.121 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:25.121 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:25.121 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.b calling monitor election 2024-04-26T04:35:25.121 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:25.121 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:25.121 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:25.121 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:25.121 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:25.121 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T04:35:25.121 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: monmap e2: 2 mons at {a=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],b=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:35:25.121 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: fsmap 2024-04-26T04:35:25.121 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:35:25.121 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mgrmap e13: y(active, since 38s) 2024-04-26T04:35:25.121 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: overall HEALTH_OK 2024-04-26T04:35:25.121 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:25.122 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:25.122 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:25.122 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:35:25.122 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:25.122 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: Deploying daemon mon.c on smithi156 2024-04-26T04:35:25.122 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:25.122 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:25.122 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: from='client.? 172.21.15.171:0/3909522661' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:35:25.122 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 ceph-mon[39130]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-26T04:35:25.122 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 bash[39092]: fc34a850791e9377c80d935c903d844b5d0ee2fe0ef2d6d2cab230f7534c9c0e 2024-04-26T04:35:25.122 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:24 smithi156 systemd[1]: Started Ceph mon.c for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:35:30.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:35:30.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: mon.a calling monitor election 2024-04-26T04:35:30.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:30.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: mon.b calling monitor election 2024-04-26T04:35:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: mon.c calling monitor election 2024-04-26T04:35:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:35:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: monmap e3: 3 mons at {a=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],b=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0],c=[v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:35:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: fsmap 2024-04-26T04:35:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:35:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: mgrmap e13: y(active, since 46s) 2024-04-26T04:35:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: overall HEALTH_OK 2024-04-26T04:35:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:30.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:35:30.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: mon.a calling monitor election 2024-04-26T04:35:30.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:30.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: mon.b calling monitor election 2024-04-26T04:35:30.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:30.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: mon.c calling monitor election 2024-04-26T04:35:30.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:30.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:30.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:35:30.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: monmap e3: 3 mons at {a=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],b=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0],c=[v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:35:30.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: fsmap 2024-04-26T04:35:30.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:35:30.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: mgrmap e13: y(active, since 46s) 2024-04-26T04:35:30.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: overall HEALTH_OK 2024-04-26T04:35:30.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:30.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:30.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:35:30.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: mon.a calling monitor election 2024-04-26T04:35:30.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:35:30.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: mon.b calling monitor election 2024-04-26T04:35:30.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:30.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: mon.c calling monitor election 2024-04-26T04:35:30.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:30.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:30.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:30.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:35:30.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: monmap e3: 3 mons at {a=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],b=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0],c=[v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:35:30.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: fsmap 2024-04-26T04:35:30.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:35:30.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: mgrmap e13: y(active, since 46s) 2024-04-26T04:35:30.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: overall HEALTH_OK 2024-04-26T04:35:30.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:30.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:30.418 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-04-26T04:35:30.418 INFO:teuthology.orchestra.run.smithi171.stdout:{"epoch":3,"fsid":"110027f2-0386-11ef-bc93-c7b262605968","modified":"2024-04-26T04:35:24.769972Z","created":"2024-04-26T04:33:43.779990Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:3300","nonce":0},{"type":"v1","addr":"172.21.15.156:6789","nonce":0}]},"addr":"172.21.15.156:6789/0","public_addr":"172.21.15.156:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:3300","nonce":0},{"type":"v1","addr":"172.21.15.171:6789","nonce":0}]},"addr":"172.21.15.171:6789/0","public_addr":"172.21.15.171:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:3301","nonce":0},{"type":"v1","addr":"172.21.15.156:6790","nonce":0}]},"addr":"172.21.15.156:6790/0","public_addr":"172.21.15.156:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-04-26T04:35:30.420 INFO:teuthology.orchestra.run.smithi171.stderr:dumped monmap epoch 3 2024-04-26T04:35:31.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:30 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:31.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:30 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:31.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:30 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:31.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:30 smithi156 ceph-mon[39130]: from='client.? 172.21.15.171:0/3048347829' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:35:31.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:30 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:31.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:30 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:31.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:30 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:31.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:30 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:31.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:30 smithi156 ceph-mon[27570]: from='client.? 172.21.15.171:0/3048347829' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:35:31.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:30 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:31.189 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:30 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:31.190 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:30 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:31.190 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:30 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:31.190 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:30 smithi171 ceph-mon[31139]: from='client.? 172.21.15.171:0/3048347829' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:35:31.190 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:30 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:35:31.214 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-26T04:35:31.217 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-26T04:35:31.462 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:35:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:32 smithi171 ceph-mon[31139]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:32 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:32 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:32 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:32 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:32 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:32.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:32 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:32.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:32 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:32 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:32 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:32 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:32 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.565 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[27570]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:32.565 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.565 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.565 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.565 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.566 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:32.566 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:32.566 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.566 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.566 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.566 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.567 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.567 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[39130]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:32.567 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.567 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.568 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.568 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.568 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:32.568 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:32.568 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.568 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.569 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.569 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:32.569 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:32 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:33.356 INFO:teuthology.orchestra.run.smithi156.stdout:# minimal ceph.conf for 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:35:33.356 INFO:teuthology.orchestra.run.smithi156.stdout:[global] 2024-04-26T04:35:33.356 INFO:teuthology.orchestra.run.smithi156.stdout: fsid = 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:35:33.356 INFO:teuthology.orchestra.run.smithi156.stdout: mon_host = [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] 2024-04-26T04:35:33.404 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[27570]: Updating smithi156:/etc/ceph/ceph.conf 2024-04-26T04:35:33.404 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[27570]: Updating smithi171:/etc/ceph/ceph.conf 2024-04-26T04:35:33.404 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[27570]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:35:33.405 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[27570]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:35:33.405 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[27570]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T04:35:33.405 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:35:33.405 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:35:33.405 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:33.405 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[27570]: Reconfiguring daemon mon.a on smithi156 2024-04-26T04:35:33.405 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:33.405 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[39130]: Updating smithi156:/etc/ceph/ceph.conf 2024-04-26T04:35:33.405 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[39130]: Updating smithi171:/etc/ceph/ceph.conf 2024-04-26T04:35:33.405 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[39130]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:35:33.405 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[39130]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:35:33.405 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[39130]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T04:35:33.406 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:35:33.406 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:35:33.406 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:33.406 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[39130]: Reconfiguring daemon mon.a on smithi156 2024-04-26T04:35:33.406 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:33.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:33 smithi171 ceph-mon[31139]: Updating smithi156:/etc/ceph/ceph.conf 2024-04-26T04:35:33.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:33 smithi171 ceph-mon[31139]: Updating smithi171:/etc/ceph/ceph.conf 2024-04-26T04:35:33.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:33 smithi171 ceph-mon[31139]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:35:33.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:33 smithi171 ceph-mon[31139]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:35:33.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:33 smithi171 ceph-mon[31139]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T04:35:33.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:35:33.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:35:33.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:33.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:33 smithi171 ceph-mon[31139]: Reconfiguring daemon mon.a on smithi156 2024-04-26T04:35:33.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:34.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:34 smithi156 ceph-mon[27570]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:34.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:34 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1302682905' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:34.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:34 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:34.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:34 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:34.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:34 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:35:34.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:34 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:35:34.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:34 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:34.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:34 smithi156 ceph-mon[39130]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:34.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:34 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1302682905' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:34.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:34 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:34.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:34 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:34.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:34 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:35:34.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:34 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:35:34.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:34 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:34.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:34 smithi171 ceph-mon[31139]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:34.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:34 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1302682905' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:34.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:34 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:34.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:34 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:34.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:34 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:35:34.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:34 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:35:34.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:34 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:34.651 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-26T04:35:34.652 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-04-26T04:35:34.652 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/etc/ceph/ceph.conf 2024-04-26T04:35:34.680 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-04-26T04:35:34.681 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:35:34.745 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-04-26T04:35:34.745 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/ceph/ceph.conf 2024-04-26T04:35:34.774 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-04-26T04:35:34.774 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:35:34.840 INFO:tasks.cephadm:Adding mgr.y on smithi156 2024-04-26T04:35:34.840 INFO:tasks.cephadm:Adding mgr.x on smithi171 2024-04-26T04:35:34.840 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph orch apply mgr '2;smithi156=y;smithi171=x' 2024-04-26T04:35:35.126 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:35:35.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:35 smithi156 ceph-mon[39130]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:35:35.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:35 smithi156 ceph-mon[39130]: Reconfiguring daemon mon.c on smithi156 2024-04-26T04:35:35.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:35 smithi156 ceph-mon[27570]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:35:35.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:35 smithi156 ceph-mon[27570]: Reconfiguring daemon mon.c on smithi156 2024-04-26T04:35:35.438 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:35 smithi171 ceph-mon[31139]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:35:35.438 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:35 smithi171 ceph-mon[31139]: Reconfiguring daemon mon.c on smithi156 2024-04-26T04:35:36.184 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:36 smithi171 ceph-mon[31139]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:36.184 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:36 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:36.184 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:36 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:36.184 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:36 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:35:36.184 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:36 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:35:36.184 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:36 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:36.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:36 smithi156 ceph-mon[39130]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:36.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:36 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:36.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:36 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:36.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:36 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:35:36.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:36 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:35:36.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:36 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:36.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:36 smithi156 ceph-mon[27570]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:36.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:36 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:36.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:36 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:36.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:36 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:35:36.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:36 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:35:36.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:36 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:37.421 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:37 smithi171 ceph-mon[31139]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:35:37.422 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:37 smithi171 ceph-mon[31139]: Reconfiguring daemon mon.b on smithi171 2024-04-26T04:35:37.495 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:37 smithi156 ceph-mon[27570]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:35:37.495 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:37 smithi156 ceph-mon[27570]: Reconfiguring daemon mon.b on smithi171 2024-04-26T04:35:37.495 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:37 smithi156 ceph-mon[39130]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:35:37.495 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:37 smithi156 ceph-mon[39130]: Reconfiguring daemon mon.b on smithi171 2024-04-26T04:35:37.526 INFO:teuthology.orchestra.run.smithi171.stdout:Scheduled mgr update... 2024-04-26T04:35:38.368 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:38 smithi171 ceph-mon[31139]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:38.368 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:38 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:38.368 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:38 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:38.368 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:38 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:38.368 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:38 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:38.368 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:38 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:38.368 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:38 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:38.368 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:38 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:38.368 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:38 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:35:38.368 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:38 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T04:35:38.368 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:38 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:35:38.368 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:38 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:38.392 DEBUG:teuthology.orchestra.run.smithi171:mgr.x> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.x.service 2024-04-26T04:35:38.394 INFO:tasks.cephadm:Deploying OSDs... 2024-04-26T04:35:38.395 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-04-26T04:35:38.395 DEBUG:teuthology.orchestra.run.smithi156:> dd if=/scratch_devs of=/dev/stdout 2024-04-26T04:35:38.413 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[27570]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:38.413 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:38.413 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:38.413 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:38.413 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:38.413 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:38.414 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:38.414 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:38.414 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:35:38.414 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T04:35:38.414 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:35:38.414 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:38.414 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[39130]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:38.414 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:38.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:38.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:38.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:38.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:38.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:38.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:38.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:35:38.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T04:35:38.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:35:38.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:38 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:38.416 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-26T04:35:38.417 DEBUG:teuthology.orchestra.run.smithi156:> stat /dev/vg_nvme/lv_1 2024-04-26T04:35:38.473 INFO:teuthology.orchestra.run.smithi156.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-26T04:35:38.473 INFO:teuthology.orchestra.run.smithi156.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:35:38.474 INFO:teuthology.orchestra.run.smithi156.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-26T04:35:38.474 INFO:teuthology.orchestra.run.smithi156.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:35:38.474 INFO:teuthology.orchestra.run.smithi156.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:35:38.474 INFO:teuthology.orchestra.run.smithi156.stdout:Access: 2024-04-26 04:35:24.255085385 +0000 2024-04-26T04:35:38.474 INFO:teuthology.orchestra.run.smithi156.stdout:Modify: 2024-04-26 04:30:21.861137846 +0000 2024-04-26T04:35:38.474 INFO:teuthology.orchestra.run.smithi156.stdout:Change: 2024-04-26 04:30:21.861137846 +0000 2024-04-26T04:35:38.474 INFO:teuthology.orchestra.run.smithi156.stdout: Birth: 2024-04-26 04:30:21.861137846 +0000 2024-04-26T04:35:38.474 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-26T04:35:38.539 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records in 2024-04-26T04:35:38.539 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records out 2024-04-26T04:35:38.539 INFO:teuthology.orchestra.run.smithi156.stderr:512 bytes copied, 9.3863e-05 s, 5.5 MB/s 2024-04-26T04:35:38.540 DEBUG:teuthology.orchestra.run.smithi156:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-26T04:35:38.597 DEBUG:teuthology.orchestra.run.smithi156:> stat /dev/vg_nvme/lv_2 2024-04-26T04:35:38.653 INFO:teuthology.orchestra.run.smithi156.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-26T04:35:38.653 INFO:teuthology.orchestra.run.smithi156.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:35:38.653 INFO:teuthology.orchestra.run.smithi156.stdout:Device: 5h/5d Inode: 683 Links: 1 2024-04-26T04:35:38.653 INFO:teuthology.orchestra.run.smithi156.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:35:38.653 INFO:teuthology.orchestra.run.smithi156.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:35:38.653 INFO:teuthology.orchestra.run.smithi156.stdout:Access: 2024-04-26 04:35:24.255085385 +0000 2024-04-26T04:35:38.653 INFO:teuthology.orchestra.run.smithi156.stdout:Modify: 2024-04-26 04:30:22.205133367 +0000 2024-04-26T04:35:38.653 INFO:teuthology.orchestra.run.smithi156.stdout:Change: 2024-04-26 04:30:22.205133367 +0000 2024-04-26T04:35:38.653 INFO:teuthology.orchestra.run.smithi156.stdout: Birth: 2024-04-26 04:30:22.205133367 +0000 2024-04-26T04:35:38.654 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-26T04:35:38.715 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records in 2024-04-26T04:35:38.716 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records out 2024-04-26T04:35:38.716 INFO:teuthology.orchestra.run.smithi156.stderr:512 bytes copied, 0.00010052 s, 5.1 MB/s 2024-04-26T04:35:38.717 DEBUG:teuthology.orchestra.run.smithi156:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-26T04:35:38.772 DEBUG:teuthology.orchestra.run.smithi156:> stat /dev/vg_nvme/lv_3 2024-04-26T04:35:38.828 INFO:teuthology.orchestra.run.smithi156.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-26T04:35:38.828 INFO:teuthology.orchestra.run.smithi156.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:35:38.828 INFO:teuthology.orchestra.run.smithi156.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-26T04:35:38.828 INFO:teuthology.orchestra.run.smithi156.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:35:38.828 INFO:teuthology.orchestra.run.smithi156.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:35:38.828 INFO:teuthology.orchestra.run.smithi156.stdout:Access: 2024-04-26 04:35:24.256085372 +0000 2024-04-26T04:35:38.828 INFO:teuthology.orchestra.run.smithi156.stdout:Modify: 2024-04-26 04:30:22.549128887 +0000 2024-04-26T04:35:38.828 INFO:teuthology.orchestra.run.smithi156.stdout:Change: 2024-04-26 04:30:22.549128887 +0000 2024-04-26T04:35:38.828 INFO:teuthology.orchestra.run.smithi156.stdout: Birth: 2024-04-26 04:30:22.549128887 +0000 2024-04-26T04:35:38.828 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-26T04:35:38.890 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records in 2024-04-26T04:35:38.890 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records out 2024-04-26T04:35:38.890 INFO:teuthology.orchestra.run.smithi156.stderr:512 bytes copied, 0.00011236 s, 4.6 MB/s 2024-04-26T04:35:38.892 DEBUG:teuthology.orchestra.run.smithi156:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-26T04:35:38.947 DEBUG:teuthology.orchestra.run.smithi156:> stat /dev/vg_nvme/lv_4 2024-04-26T04:35:39.003 INFO:teuthology.orchestra.run.smithi156.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-26T04:35:39.004 INFO:teuthology.orchestra.run.smithi156.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:35:39.004 INFO:teuthology.orchestra.run.smithi156.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-04-26T04:35:39.004 INFO:teuthology.orchestra.run.smithi156.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:35:39.004 INFO:teuthology.orchestra.run.smithi156.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:35:39.004 INFO:teuthology.orchestra.run.smithi156.stdout:Access: 2024-04-26 04:35:24.256085372 +0000 2024-04-26T04:35:39.004 INFO:teuthology.orchestra.run.smithi156.stdout:Modify: 2024-04-26 04:30:22.891124434 +0000 2024-04-26T04:35:39.004 INFO:teuthology.orchestra.run.smithi156.stdout:Change: 2024-04-26 04:30:22.891124434 +0000 2024-04-26T04:35:39.004 INFO:teuthology.orchestra.run.smithi156.stdout: Birth: 2024-04-26 04:30:22.891124434 +0000 2024-04-26T04:35:39.004 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-26T04:35:39.066 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records in 2024-04-26T04:35:39.066 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records out 2024-04-26T04:35:39.066 INFO:teuthology.orchestra.run.smithi156.stderr:512 bytes copied, 8.6501e-05 s, 5.9 MB/s 2024-04-26T04:35:39.067 DEBUG:teuthology.orchestra.run.smithi156:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-26T04:35:39.123 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-04-26T04:35:39.123 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/scratch_devs of=/dev/stdout 2024-04-26T04:35:39.139 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-26T04:35:39.139 DEBUG:teuthology.orchestra.run.smithi171:> stat /dev/vg_nvme/lv_1 2024-04-26T04:35:39.196 INFO:teuthology.orchestra.run.smithi171.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-26T04:35:39.196 INFO:teuthology.orchestra.run.smithi171.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:35:39.196 INFO:teuthology.orchestra.run.smithi171.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-04-26T04:35:39.196 INFO:teuthology.orchestra.run.smithi171.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:35:39.196 INFO:teuthology.orchestra.run.smithi171.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:35:39.196 INFO:teuthology.orchestra.run.smithi171.stdout:Access: 2024-04-26 04:35:15.500921601 +0000 2024-04-26T04:35:39.196 INFO:teuthology.orchestra.run.smithi171.stdout:Modify: 2024-04-26 04:30:21.854496107 +0000 2024-04-26T04:35:39.196 INFO:teuthology.orchestra.run.smithi171.stdout:Change: 2024-04-26 04:30:21.854496107 +0000 2024-04-26T04:35:39.197 INFO:teuthology.orchestra.run.smithi171.stdout: Birth: 2024-04-26 04:30:21.854496107 +0000 2024-04-26T04:35:39.197 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-26T04:35:39.259 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records in 2024-04-26T04:35:39.259 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records out 2024-04-26T04:35:39.259 INFO:teuthology.orchestra.run.smithi171.stderr:512 bytes copied, 0.000209978 s, 2.4 MB/s 2024-04-26T04:35:39.261 DEBUG:teuthology.orchestra.run.smithi171:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-26T04:35:39.317 DEBUG:teuthology.orchestra.run.smithi171:> stat /dev/vg_nvme/lv_2 2024-04-26T04:35:39.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:39 smithi156 ceph-mon[39130]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi156=y;smithi171=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:35:39.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:39 smithi156 ceph-mon[39130]: Saving service mgr spec with placement smithi156=y;smithi171=x;count:2 2024-04-26T04:35:39.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:39 smithi156 ceph-mon[39130]: Deploying daemon mgr.x on smithi171 2024-04-26T04:35:39.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:39 smithi156 ceph-mon[27570]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi156=y;smithi171=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:35:39.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:39 smithi156 ceph-mon[27570]: Saving service mgr spec with placement smithi156=y;smithi171=x;count:2 2024-04-26T04:35:39.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:39 smithi156 ceph-mon[27570]: Deploying daemon mgr.x on smithi171 2024-04-26T04:35:39.371 INFO:teuthology.orchestra.run.smithi171.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-26T04:35:39.371 INFO:teuthology.orchestra.run.smithi171.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:35:39.372 INFO:teuthology.orchestra.run.smithi171.stdout:Device: 5h/5d Inode: 683 Links: 1 2024-04-26T04:35:39.372 INFO:teuthology.orchestra.run.smithi171.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:35:39.372 INFO:teuthology.orchestra.run.smithi171.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:35:39.372 INFO:teuthology.orchestra.run.smithi171.stdout:Access: 2024-04-26 04:35:15.501921582 +0000 2024-04-26T04:35:39.372 INFO:teuthology.orchestra.run.smithi171.stdout:Modify: 2024-04-26 04:30:22.218488658 +0000 2024-04-26T04:35:39.372 INFO:teuthology.orchestra.run.smithi171.stdout:Change: 2024-04-26 04:30:22.218488658 +0000 2024-04-26T04:35:39.372 INFO:teuthology.orchestra.run.smithi171.stdout: Birth: 2024-04-26 04:30:22.218488658 +0000 2024-04-26T04:35:39.372 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-26T04:35:39.431 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:39 smithi171 ceph-mon[31139]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi156=y;smithi171=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:35:39.431 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:39 smithi171 ceph-mon[31139]: Saving service mgr spec with placement smithi156=y;smithi171=x;count:2 2024-04-26T04:35:39.431 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:39 smithi171 ceph-mon[31139]: Deploying daemon mgr.x on smithi171 2024-04-26T04:35:39.433 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records in 2024-04-26T04:35:39.433 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records out 2024-04-26T04:35:39.433 INFO:teuthology.orchestra.run.smithi171.stderr:512 bytes copied, 0.000193033 s, 2.7 MB/s 2024-04-26T04:35:39.436 DEBUG:teuthology.orchestra.run.smithi171:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-26T04:35:39.491 DEBUG:teuthology.orchestra.run.smithi171:> stat /dev/vg_nvme/lv_3 2024-04-26T04:35:39.546 INFO:teuthology.orchestra.run.smithi171.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-26T04:35:39.546 INFO:teuthology.orchestra.run.smithi171.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:35:39.546 INFO:teuthology.orchestra.run.smithi171.stdout:Device: 5h/5d Inode: 702 Links: 1 2024-04-26T04:35:39.546 INFO:teuthology.orchestra.run.smithi171.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:35:39.546 INFO:teuthology.orchestra.run.smithi171.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:35:39.546 INFO:teuthology.orchestra.run.smithi171.stdout:Access: 2024-04-26 04:35:15.501921582 +0000 2024-04-26T04:35:39.547 INFO:teuthology.orchestra.run.smithi171.stdout:Modify: 2024-04-26 04:30:22.537482131 +0000 2024-04-26T04:35:39.547 INFO:teuthology.orchestra.run.smithi171.stdout:Change: 2024-04-26 04:30:22.537482131 +0000 2024-04-26T04:35:39.547 INFO:teuthology.orchestra.run.smithi171.stdout: Birth: 2024-04-26 04:30:22.537482131 +0000 2024-04-26T04:35:39.547 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-26T04:35:39.608 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records in 2024-04-26T04:35:39.609 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records out 2024-04-26T04:35:39.609 INFO:teuthology.orchestra.run.smithi171.stderr:512 bytes copied, 0.00019243 s, 2.7 MB/s 2024-04-26T04:35:39.610 DEBUG:teuthology.orchestra.run.smithi171:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-26T04:35:39.666 DEBUG:teuthology.orchestra.run.smithi171:> stat /dev/vg_nvme/lv_4 2024-04-26T04:35:39.721 INFO:teuthology.orchestra.run.smithi171.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-26T04:35:39.721 INFO:teuthology.orchestra.run.smithi171.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:35:39.721 INFO:teuthology.orchestra.run.smithi171.stdout:Device: 5h/5d Inode: 721 Links: 1 2024-04-26T04:35:39.721 INFO:teuthology.orchestra.run.smithi171.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:35:39.721 INFO:teuthology.orchestra.run.smithi171.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:35:39.721 INFO:teuthology.orchestra.run.smithi171.stdout:Access: 2024-04-26 04:35:15.501921582 +0000 2024-04-26T04:35:39.721 INFO:teuthology.orchestra.run.smithi171.stdout:Modify: 2024-04-26 04:30:22.880475113 +0000 2024-04-26T04:35:39.721 INFO:teuthology.orchestra.run.smithi171.stdout:Change: 2024-04-26 04:30:22.880475113 +0000 2024-04-26T04:35:39.721 INFO:teuthology.orchestra.run.smithi171.stdout: Birth: 2024-04-26 04:30:22.880475113 +0000 2024-04-26T04:35:39.722 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-26T04:35:39.812 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records in 2024-04-26T04:35:39.812 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records out 2024-04-26T04:35:39.812 INFO:teuthology.orchestra.run.smithi171.stderr:512 bytes copied, 0.000185641 s, 2.8 MB/s 2024-04-26T04:35:39.813 DEBUG:teuthology.orchestra.run.smithi171:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-26T04:35:39.828 INFO:tasks.cephadm:Deploying osd.0 on smithi156 with /dev/vg_nvme/lv_4... 2024-04-26T04:35:39.828 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-26T04:35:40.069 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:35:40.414 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:40 smithi171 ceph-mon[31139]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:40.415 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:40 smithi171 podman[33225]: 2024-04-26 04:35:40.227345757 +0000 UTC m=+0.124067360 container create 77edd00db79e1b66482692db88cb34e7aa8b704318102407cf7b54eb6c7626b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, RELEASE=quincy-c61472c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:35:40.415 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:40 smithi171 podman[33225]: 2024-04-26 04:35:40.413627736 +0000 UTC m=+0.310349302 container init 77edd00db79e1b66482692db88cb34e7aa8b704318102407cf7b54eb6c7626b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:35:40.437 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:40 smithi156 ceph-mon[27570]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:40.437 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:40 smithi156 ceph-mon[39130]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:40.746 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:40 smithi171 podman[33225]: 2024-04-26 04:35:40.419125107 +0000 UTC m=+0.315846674 container start 77edd00db79e1b66482692db88cb34e7aa8b704318102407cf7b54eb6c7626b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True) 2024-04-26T04:35:40.746 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:40 smithi171 ceph-mgr[33263]: -- 172.21.15.171:0/1539337871 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5602214269c0 con 0x5602213fc800 2024-04-26T04:35:40.746 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:40 smithi171 bash[33225]: 77edd00db79e1b66482692db88cb34e7aa8b704318102407cf7b54eb6c7626b7 2024-04-26T04:35:40.746 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:40 smithi171 systemd[1]: Started Ceph mgr.x for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:35:40.746 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:40.570+0000 7fc3e20be1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:35:40.746 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:40.745+0000 7fc3e20be1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:35:40.999 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:40.852+0000 7fc3e20be1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:35:41.471 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:41 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:41.183+0000 7fc3e20be1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:35:41.471 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:41 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:41.265+0000 7fc3e20be1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:35:41.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:41 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:41.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:41 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:41.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:41 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:41.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:41 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:41.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:41 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:41.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:41 smithi171 ceph-mon[31139]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:41.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:41 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:41.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:41 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:41.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:41 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:41.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:41 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:41.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:41 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:41.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:41 smithi156 ceph-mon[27570]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:41.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:41 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:41.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:41 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:41.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:41 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:41.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:41 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:41.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:41 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:41.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:41 smithi156 ceph-mon[39130]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:41.939 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-26T04:35:41.939 INFO:teuthology.orchestra.run.smithi156.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-26T04:35:41.939 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10+0 records in 2024-04-26T04:35:41.939 INFO:teuthology.orchestra.run.smithi156.stderr:10+0 records out 2024-04-26T04:35:41.939 INFO:teuthology.orchestra.run.smithi156.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00842804 s, 1.2 GB/s 2024-04-26T04:35:41.939 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping successful for: 2024-04-26T04:35:42.637 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi156:vg_nvme/lv_4 2024-04-26T04:35:42.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:42.449+0000 7fc3e20be1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:35:42.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:42.612+0000 7fc3e20be1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:35:42.881 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:42.881 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:42.881 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:42.881 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:42.881 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:42.881 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[39130]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T04:35:42.881 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:35:42.881 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:35:42.881 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:42.882 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[39130]: Reconfiguring daemon mgr.y on smithi156 2024-04-26T04:35:42.882 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:42.882 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:42.882 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:42.882 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:42.882 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:42.882 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[27570]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T04:35:42.882 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:35:42.882 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:35:42.882 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:42.883 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:42 smithi156 ceph-mon[27570]: Reconfiguring daemon mgr.y on smithi156 2024-04-26T04:35:43.037 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:42.778+0000 7fc3e20be1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:35:43.037 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:43 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:43.036+0000 7fc3e20be1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:35:43.037 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:42 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:43.037 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:42 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:43.037 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:42 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:43.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:42 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:43.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:42 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:43.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:42 smithi171 ceph-mon[31139]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T04:35:43.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:42 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:35:43.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:42 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:35:43.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:42 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:43.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:42 smithi171 ceph-mon[31139]: Reconfiguring daemon mgr.y on smithi156 2024-04-26T04:35:43.490 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:43 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:43.123+0000 7fc3e20be1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:35:43.490 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:43 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:43.214+0000 7fc3e20be1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:35:43.731 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:35:43.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:43 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:43.488+0000 7fc3e20be1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:35:43.976 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:43 smithi156 ceph-mon[27570]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:43.976 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:43 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:43.976 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:43 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:43.976 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:43 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:43.977 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:43 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:43.977 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:43 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:43.977 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:43 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:43.977 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:43 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:43.977 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:43 smithi156 ceph-mon[39130]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:43.977 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:43 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:43.977 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:43 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:43.978 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:43 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:43.978 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:43 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:43.978 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:43 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:43.978 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:43 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:43.978 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:43 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:44.249 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:43 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:43.942+0000 7fc3e20be1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:35:44.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:43 smithi171 ceph-mon[31139]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:44.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:43 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:44.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:43 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:44.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:43 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:44.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:43 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:44.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:43 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:44.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:43 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:44.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:43 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:35:44.728 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:44 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:44.463+0000 7fc3e20be1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:35:44.728 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:44 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:44.555+0000 7fc3e20be1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:35:44.728 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:44 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:44.726+0000 7fc3e20be1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:35:45.268 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:45.012+0000 7fc3e20be1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:35:45.269 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:45.267+0000 7fc3e20be1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:35:45.632 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:45.501+0000 7fc3e20be1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:35:45.921 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:45.630+0000 7fc3e20be1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:35:45.921 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:45.721+0000 7fc3e20be1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:35:45.922 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:45.833+0000 7fc3e20be1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:35:45.922 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:45.920+0000 7fc3e20be1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:35:46.148 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:45 smithi156 ceph-mon[27570]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:46.148 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:45 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:35:46.148 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:45 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:35:46.148 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:45 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:46.148 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:45 smithi156 ceph-mon[39130]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:46.148 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:45 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:35:46.148 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:45 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:35:46.148 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:45 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:46.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:45 smithi171 ceph-mon[31139]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:46.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:45 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:35:46.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:45 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:35:46.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:45 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:46.999 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:46 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:46.710+0000 7fc3e20be1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:35:47.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:35:46 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:35:46.900+0000 7fc3e20be1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:35:47.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:46 smithi171 ceph-mon[31139]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:35:47.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:46 smithi171 ceph-mon[31139]: Standby manager daemon x started 2024-04-26T04:35:47.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:46 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/3822513793' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:35:47.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:46 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/3822513793' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:35:47.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:46 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/3822513793' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:35:47.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:46 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/3822513793' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:35:47.145 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:46 smithi156 ceph-mon[27570]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:35:47.145 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:46 smithi156 ceph-mon[27570]: Standby manager daemon x started 2024-04-26T04:35:47.145 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:46 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/3822513793' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:35:47.145 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:46 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/3822513793' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:35:47.145 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:46 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/3822513793' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:35:47.145 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:46 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/3822513793' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:35:47.145 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:46 smithi156 ceph-mon[39130]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:35:47.146 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:46 smithi156 ceph-mon[39130]: Standby manager daemon x started 2024-04-26T04:35:47.146 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:46 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/3822513793' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:35:47.146 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:46 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/3822513793' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:35:47.146 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:46 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/3822513793' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:35:47.146 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:46 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/3822513793' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:35:48.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:47 smithi171 ceph-mon[31139]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:48.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:47 smithi171 ceph-mon[31139]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-26T04:35:48.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:47 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:35:48.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:47 smithi156 ceph-mon[27570]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:48.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:47 smithi156 ceph-mon[27570]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-26T04:35:48.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:47 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:35:48.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:47 smithi156 ceph-mon[39130]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:48.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:47 smithi156 ceph-mon[39130]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-26T04:35:48.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:47 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:35:49.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:48 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1736804832' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6b59fe49-0273-4115-91d9-6faea7d37616"}]: dispatch 2024-04-26T04:35:49.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:48 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1736804832' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6b59fe49-0273-4115-91d9-6faea7d37616"}]': finished 2024-04-26T04:35:49.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:48 smithi171 ceph-mon[31139]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T04:35:49.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:48 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:49.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:48 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3786041137' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:35:49.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:48 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1736804832' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6b59fe49-0273-4115-91d9-6faea7d37616"}]: dispatch 2024-04-26T04:35:49.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:48 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1736804832' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6b59fe49-0273-4115-91d9-6faea7d37616"}]': finished 2024-04-26T04:35:49.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:48 smithi156 ceph-mon[27570]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T04:35:49.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:48 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:49.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:48 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3786041137' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:35:49.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:48 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1736804832' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6b59fe49-0273-4115-91d9-6faea7d37616"}]: dispatch 2024-04-26T04:35:49.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:48 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1736804832' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6b59fe49-0273-4115-91d9-6faea7d37616"}]': finished 2024-04-26T04:35:49.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:48 smithi156 ceph-mon[39130]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T04:35:49.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:48 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:49.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:48 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3786041137' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:35:50.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:49 smithi171 ceph-mon[31139]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:50.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:49 smithi156 ceph-mon[27570]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:50.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:49 smithi156 ceph-mon[39130]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:52.137 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:51 smithi156 ceph-mon[27570]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:52.137 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:51 smithi156 ceph-mon[39130]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:52.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:51 smithi171 ceph-mon[31139]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:54.197 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:53 smithi156 ceph-mon[27570]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:54.197 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:53 smithi156 ceph-mon[39130]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:54.244 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:53 smithi171 ceph-mon[31139]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:55.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:54 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:35:55.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:54 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:55.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:54 smithi171 ceph-mon[31139]: Deploying daemon osd.0 on smithi156 2024-04-26T04:35:55.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:54 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:35:55.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:54 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:55.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:54 smithi156 ceph-mon[27570]: Deploying daemon osd.0 on smithi156 2024-04-26T04:35:55.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:54 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:35:55.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:54 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:55.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:54 smithi156 ceph-mon[39130]: Deploying daemon osd.0 on smithi156 2024-04-26T04:35:56.181 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:55 smithi156 ceph-mon[27570]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:56.181 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:55 smithi156 ceph-mon[39130]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:56.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:55 smithi171 ceph-mon[31139]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:58.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:35:57 smithi171 ceph-mon[31139]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:58.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:35:57 smithi156 ceph-mon[27570]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:58.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:35:57 smithi156 ceph-mon[39130]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:36:00.696 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:00 smithi156 ceph-mon[27570]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:36:00.697 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:00 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:00.697 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:00 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:00.697 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:00 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:00.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:00 smithi156 ceph-mon[39130]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:36:00.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:00 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:00.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:00 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:00.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:00 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:00.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:00 smithi171 ceph-mon[31139]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:36:00.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:00 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:00.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:00 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:00.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:00 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:02.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:02 smithi156 ceph-mon[39130]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:36:02.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:02 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:02.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:02 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:02.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:02 smithi156 ceph-mon[27570]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:36:02.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:02 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:02.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:02 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:02.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:02 smithi171 ceph-mon[31139]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:36:02.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:02 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:02.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:02 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:02.990 INFO:teuthology.orchestra.run.smithi156.stdout:Created osd(s) 0 on host 'smithi156' 2024-04-26T04:36:03.560 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:03 smithi156 ceph-mon[27570]: from='osd.0 [v2:172.21.15.156:6802/3436276177,v1:172.21.15.156:6803/3436276177]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:36:03.560 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:03 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:03.560 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:03 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:03.560 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:03 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:03.560 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:03 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:03.560 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:03 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:03.560 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:03 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:03.561 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:03 smithi156 ceph-mon[39130]: from='osd.0 [v2:172.21.15.156:6802/3436276177,v1:172.21.15.156:6803/3436276177]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:36:03.561 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:03 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:03.561 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:03 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:03.561 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:03 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:03.561 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:03 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:03.561 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:03 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:03.561 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:03 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:03.585 DEBUG:teuthology.orchestra.run.smithi156:osd.0> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.0.service 2024-04-26T04:36:03.588 INFO:tasks.cephadm:Deploying osd.1 on smithi156 with /dev/vg_nvme/lv_3... 2024-04-26T04:36:03.588 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-26T04:36:03.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:03 smithi171 ceph-mon[31139]: from='osd.0 [v2:172.21.15.156:6802/3436276177,v1:172.21.15.156:6803/3436276177]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:36:03.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:03 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:03.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:03 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:03.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:03 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:03.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:03 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:03.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:03 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:03.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:03 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:04.382 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:36:04.734 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:04 smithi156 ceph-mon[27570]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:36:04.734 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:04 smithi156 ceph-mon[27570]: from='osd.0 [v2:172.21.15.156:6802/3436276177,v1:172.21.15.156:6803/3436276177]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:36:04.734 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:04 smithi156 ceph-mon[27570]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T04:36:04.734 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:04 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:36:04.734 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:04 smithi156 ceph-mon[27570]: from='osd.0 [v2:172.21.15.156:6802/3436276177,v1:172.21.15.156:6803/3436276177]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:36:04.734 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:04 smithi156 ceph-mon[39130]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:36:04.735 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:04 smithi156 ceph-mon[39130]: from='osd.0 [v2:172.21.15.156:6802/3436276177,v1:172.21.15.156:6803/3436276177]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:36:04.735 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:04 smithi156 ceph-mon[39130]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T04:36:04.735 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:04 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:36:04.735 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:04 smithi156 ceph-mon[39130]: from='osd.0 [v2:172.21.15.156:6802/3436276177,v1:172.21.15.156:6803/3436276177]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:36:04.735 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:36:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0[43431]: 2024-04-26T04:36:04.377+0000 7fcdac5b5700 -1 osd.0 0 waiting for initial osdmap 2024-04-26T04:36:04.735 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:36:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0[43431]: 2024-04-26T04:36:04.379+0000 7fcda8bb3700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:36:04.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:04 smithi171 ceph-mon[31139]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:36:04.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:04 smithi171 ceph-mon[31139]: from='osd.0 [v2:172.21.15.156:6802/3436276177,v1:172.21.15.156:6803/3436276177]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:36:04.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:04 smithi171 ceph-mon[31139]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T04:36:04.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:04 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:36:04.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:04 smithi171 ceph-mon[31139]: from='osd.0 [v2:172.21.15.156:6802/3436276177,v1:172.21.15.156:6803/3436276177]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:36:05.412 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:05 smithi156 ceph-mon[27570]: from='osd.0 [v2:172.21.15.156:6802/3436276177,v1:172.21.15.156:6803/3436276177]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-04-26T04:36:05.412 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:05 smithi156 ceph-mon[27570]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T04:36:05.412 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:05 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:36:05.412 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:05 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:36:05.412 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:05 smithi156 ceph-mon[39130]: from='osd.0 [v2:172.21.15.156:6802/3436276177,v1:172.21.15.156:6803/3436276177]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-04-26T04:36:05.412 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:05 smithi156 ceph-mon[39130]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T04:36:05.412 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:05 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:36:05.413 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:05 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:36:05.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:05 smithi171 ceph-mon[31139]: from='osd.0 [v2:172.21.15.156:6802/3436276177,v1:172.21.15.156:6803/3436276177]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-04-26T04:36:05.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:05 smithi171 ceph-mon[31139]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T04:36:05.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:05 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:36:05.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:05 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:36:06.640 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:06 smithi156 ceph-mon[27570]: purged_snaps scrub starts 2024-04-26T04:36:06.640 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:06 smithi156 ceph-mon[27570]: purged_snaps scrub ok 2024-04-26T04:36:06.640 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:06 smithi156 ceph-mon[27570]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:36:06.640 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:06 smithi156 ceph-mon[27570]: osd.0 [v2:172.21.15.156:6802/3436276177,v1:172.21.15.156:6803/3436276177] boot 2024-04-26T04:36:06.640 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:06 smithi156 ceph-mon[27570]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T04:36:06.640 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:06 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:36:06.641 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:06 smithi156 ceph-mon[39130]: purged_snaps scrub starts 2024-04-26T04:36:06.641 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:06 smithi156 ceph-mon[39130]: purged_snaps scrub ok 2024-04-26T04:36:06.641 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:06 smithi156 ceph-mon[39130]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:36:06.641 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:06 smithi156 ceph-mon[39130]: osd.0 [v2:172.21.15.156:6802/3436276177,v1:172.21.15.156:6803/3436276177] boot 2024-04-26T04:36:06.641 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:06 smithi156 ceph-mon[39130]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T04:36:06.641 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:06 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:36:06.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:06 smithi171 ceph-mon[31139]: purged_snaps scrub starts 2024-04-26T04:36:06.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:06 smithi171 ceph-mon[31139]: purged_snaps scrub ok 2024-04-26T04:36:06.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:06 smithi171 ceph-mon[31139]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:36:06.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:06 smithi171 ceph-mon[31139]: osd.0 [v2:172.21.15.156:6802/3436276177,v1:172.21.15.156:6803/3436276177] boot 2024-04-26T04:36:06.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:06 smithi171 ceph-mon[31139]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T04:36:06.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:06 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:36:07.016 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-26T04:36:07.016 INFO:teuthology.orchestra.run.smithi156.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-26T04:36:07.016 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10+0 records in 2024-04-26T04:36:07.016 INFO:teuthology.orchestra.run.smithi156.stderr:10+0 records out 2024-04-26T04:36:07.016 INFO:teuthology.orchestra.run.smithi156.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0100336 s, 1.0 GB/s 2024-04-26T04:36:07.016 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping successful for: 2024-04-26T04:36:07.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[27570]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T04:36:07.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[27570]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:07.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[27570]: Detected new or changed devices on smithi156 2024-04-26T04:36:07.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:07.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:07.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:07.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[27570]: Adjusting osd_memory_target on smithi156 to 16064M 2024-04-26T04:36:07.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:07.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:07.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:07.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:07.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[39130]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T04:36:07.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[39130]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:07.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[39130]: Detected new or changed devices on smithi156 2024-04-26T04:36:07.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:07.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:07.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:07.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[39130]: Adjusting osd_memory_target on smithi156 to 16064M 2024-04-26T04:36:07.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:07.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:07.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:07.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:07 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:07.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:07 smithi171 ceph-mon[31139]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T04:36:07.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:07 smithi171 ceph-mon[31139]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:07.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:07 smithi171 ceph-mon[31139]: Detected new or changed devices on smithi156 2024-04-26T04:36:07.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:07 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:07.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:07 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:07.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:07 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:07.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:07 smithi171 ceph-mon[31139]: Adjusting osd_memory_target on smithi156 to 16064M 2024-04-26T04:36:07.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:07 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:07.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:07 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:07.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:07 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:07.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:07 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:07.944 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi156:vg_nvme/lv_3 2024-04-26T04:36:08.194 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:36:10.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:09 smithi171 ceph-mon[31139]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:10.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:09 smithi156 ceph-mon[27570]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:10.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:09 smithi156 ceph-mon[39130]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:11.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:10 smithi156 ceph-mon[27570]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:36:11.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:10 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:36:11.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:10 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:36:11.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:10 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:11.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:10 smithi156 ceph-mon[39130]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:36:11.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:10 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:36:11.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:10 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:36:11.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:10 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:11.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:10 smithi171 ceph-mon[31139]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:36:11.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:10 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:36:11.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:10 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:36:11.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:10 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:12.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:11 smithi156 ceph-mon[27570]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:11 smithi156 ceph-mon[39130]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:12.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:11 smithi171 ceph-mon[31139]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:13.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:12 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1797736308' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c9673007-264f-4d7c-8234-bfd514e70d2e"}]: dispatch 2024-04-26T04:36:13.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:12 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1797736308' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c9673007-264f-4d7c-8234-bfd514e70d2e"}]': finished 2024-04-26T04:36:13.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:12 smithi171 ceph-mon[31139]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T04:36:13.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:12 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:36:13.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:12 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1797736308' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c9673007-264f-4d7c-8234-bfd514e70d2e"}]: dispatch 2024-04-26T04:36:13.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:12 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1797736308' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c9673007-264f-4d7c-8234-bfd514e70d2e"}]': finished 2024-04-26T04:36:13.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:12 smithi156 ceph-mon[27570]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T04:36:13.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:12 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:36:13.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:12 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1797736308' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c9673007-264f-4d7c-8234-bfd514e70d2e"}]: dispatch 2024-04-26T04:36:13.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:12 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1797736308' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c9673007-264f-4d7c-8234-bfd514e70d2e"}]': finished 2024-04-26T04:36:13.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:12 smithi156 ceph-mon[39130]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T04:36:13.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:12 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:36:14.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:13 smithi171 ceph-mon[31139]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:14.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:13 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3225465612' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:36:14.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:13 smithi156 ceph-mon[27570]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:14.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:13 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3225465612' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:36:14.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:13 smithi156 ceph-mon[39130]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:14.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:13 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3225465612' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:36:16.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:15 smithi171 ceph-mon[31139]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:16.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:15 smithi156 ceph-mon[27570]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:16.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:15 smithi156 ceph-mon[39130]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:18.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:17 smithi171 ceph-mon[31139]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:18.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:17 smithi156 ceph-mon[27570]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:18.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:17 smithi156 ceph-mon[39130]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:19.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:18 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:36:19.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:18 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:19.254 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:18 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:36:19.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:18 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:19.255 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:18 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:36:19.255 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:18 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:20.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:19 smithi171 ceph-mon[31139]: Deploying daemon osd.1 on smithi156 2024-04-26T04:36:20.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:19 smithi171 ceph-mon[31139]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:20.329 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:19 smithi156 ceph-mon[27570]: Deploying daemon osd.1 on smithi156 2024-04-26T04:36:20.329 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:19 smithi156 ceph-mon[27570]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:20.329 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:19 smithi156 ceph-mon[39130]: Deploying daemon osd.1 on smithi156 2024-04-26T04:36:20.330 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:19 smithi156 ceph-mon[39130]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:22.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:21 smithi171 ceph-mon[31139]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:22.332 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:21 smithi156 ceph-mon[27570]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:22.332 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:21 smithi156 ceph-mon[39130]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:24.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:23 smithi156 ceph-mon[39130]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:24.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:23 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:24.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:23 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:24.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:24 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:24.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:23 smithi156 ceph-mon[27570]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:24.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:24 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:24.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:24 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:24.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:24 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:24.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:23 smithi171 ceph-mon[31139]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:24.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:23 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:24.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:23 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:24.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:23 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:26.243 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:26 smithi156 ceph-mon[27570]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:26.243 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:26 smithi156 ceph-mon[39130]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:26.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:26 smithi171 ceph-mon[31139]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:27.032 INFO:teuthology.orchestra.run.smithi156.stdout:Created osd(s) 1 on host 'smithi156' 2024-04-26T04:36:27.677 DEBUG:teuthology.orchestra.run.smithi156:osd.1> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.1.service 2024-04-26T04:36:27.682 INFO:tasks.cephadm:Deploying osd.2 on smithi156 with /dev/vg_nvme/lv_2... 2024-04-26T04:36:27.682 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-26T04:36:27.984 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:27 smithi156 ceph-mon[39130]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:27.994 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:36:28.243 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:27 smithi156 ceph-mon[27570]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:28.243 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:27 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:28.243 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:27 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:28.244 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:27 smithi156 ceph-mon[27570]: from='osd.1 [v2:172.21.15.156:6810/273749030,v1:172.21.15.156:6811/273749030]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:36:28.244 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:27 smithi156 ceph-mon[27570]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:36:28.244 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:27 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:28.244 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:27 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:28.244 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:27 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:28.244 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:27 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:28.244 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:27 smithi156 ceph-mon[39130]: from='osd.1 [v2:172.21.15.156:6810/273749030,v1:172.21.15.156:6811/273749030]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:36:28.244 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:27 smithi156 ceph-mon[39130]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:36:28.244 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:27 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:28.244 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:27 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:28.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:27 smithi171 ceph-mon[31139]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:28.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:27 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:28.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:27 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:28.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:27 smithi171 ceph-mon[31139]: from='osd.1 [v2:172.21.15.156:6810/273749030,v1:172.21.15.156:6811/273749030]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:36:28.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:27 smithi171 ceph-mon[31139]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:36:28.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:27 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:28.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:27 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:29.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:29 smithi156 ceph-mon[27570]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:36:29.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:29 smithi156 ceph-mon[27570]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T04:36:29.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:36:29.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:29 smithi156 ceph-mon[27570]: from='osd.1 [v2:172.21.15.156:6810/273749030,v1:172.21.15.156:6811/273749030]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:36:29.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:29 smithi156 ceph-mon[27570]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:36:29.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:29 smithi156 ceph-mon[39130]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:36:29.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:29 smithi156 ceph-mon[39130]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T04:36:29.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:36:29.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:29 smithi156 ceph-mon[39130]: from='osd.1 [v2:172.21.15.156:6810/273749030,v1:172.21.15.156:6811/273749030]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:36:29.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:29 smithi156 ceph-mon[39130]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:36:29.094 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:36:29 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1[49950]: 2024-04-26T04:36:29.012+0000 7ff04c2f3700 -1 osd.1 0 waiting for initial osdmap 2024-04-26T04:36:29.094 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:36:29 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1[49950]: 2024-04-26T04:36:29.014+0000 7ff0450ea700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:36:29.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:29 smithi171 ceph-mon[31139]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:36:29.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:29 smithi171 ceph-mon[31139]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T04:36:29.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:36:29.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:29 smithi171 ceph-mon[31139]: from='osd.1 [v2:172.21.15.156:6810/273749030,v1:172.21.15.156:6811/273749030]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:36:29.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:29 smithi171 ceph-mon[31139]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:36:30.069 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:30 smithi156 ceph-mon[39130]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:30.069 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:30 smithi156 ceph-mon[39130]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-04-26T04:36:30.069 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:30 smithi156 ceph-mon[39130]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T04:36:30.069 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:30 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:36:30.070 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:30 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:36:30.070 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:30 smithi156 ceph-mon[27570]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:30.070 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:30 smithi156 ceph-mon[27570]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-04-26T04:36:30.070 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:30 smithi156 ceph-mon[27570]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T04:36:30.070 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:30 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:36:30.070 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:30 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:36:30.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:30 smithi171 ceph-mon[31139]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:30.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:30 smithi171 ceph-mon[31139]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-04-26T04:36:30.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:30 smithi171 ceph-mon[31139]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T04:36:30.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:30 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:36:30.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:30 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:36:31.096 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-26T04:36:31.096 INFO:teuthology.orchestra.run.smithi156.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-26T04:36:31.096 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10+0 records in 2024-04-26T04:36:31.096 INFO:teuthology.orchestra.run.smithi156.stderr:10+0 records out 2024-04-26T04:36:31.096 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00970622 s, 1.1 GB/s 2024-04-26T04:36:31.096 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping successful for: 2024-04-26T04:36:31.118 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:31 smithi156 ceph-mon[27570]: purged_snaps scrub starts 2024-04-26T04:36:31.118 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:31 smithi156 ceph-mon[27570]: purged_snaps scrub ok 2024-04-26T04:36:31.118 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:31 smithi156 ceph-mon[27570]: osd.1 [v2:172.21.15.156:6810/273749030,v1:172.21.15.156:6811/273749030] boot 2024-04-26T04:36:31.118 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:31 smithi156 ceph-mon[27570]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T04:36:31.118 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:31 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:36:31.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:31 smithi156 ceph-mon[39130]: purged_snaps scrub starts 2024-04-26T04:36:31.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:31 smithi156 ceph-mon[39130]: purged_snaps scrub ok 2024-04-26T04:36:31.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:31 smithi156 ceph-mon[39130]: osd.1 [v2:172.21.15.156:6810/273749030,v1:172.21.15.156:6811/273749030] boot 2024-04-26T04:36:31.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:31 smithi156 ceph-mon[39130]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T04:36:31.119 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:31 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:36:31.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:31 smithi171 ceph-mon[31139]: purged_snaps scrub starts 2024-04-26T04:36:31.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:31 smithi171 ceph-mon[31139]: purged_snaps scrub ok 2024-04-26T04:36:31.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:31 smithi171 ceph-mon[31139]: osd.1 [v2:172.21.15.156:6810/273749030,v1:172.21.15.156:6811/273749030] boot 2024-04-26T04:36:31.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:31 smithi171 ceph-mon[31139]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T04:36:31.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:31 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:36:31.801 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi156:vg_nvme/lv_2 2024-04-26T04:36:32.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:32 smithi156 ceph-mon[27570]: pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:32.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:32 smithi156 ceph-mon[27570]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T04:36:32.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:32 smithi156 ceph-mon[39130]: pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:32.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:32 smithi156 ceph-mon[39130]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T04:36:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:32 smithi171 ceph-mon[31139]: pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:36:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:32 smithi171 ceph-mon[31139]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T04:36:32.527 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:36:33.669 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[27570]: Detected new or changed devices on smithi156 2024-04-26T04:36:33.669 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:33.669 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:33.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:33.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:33.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[27570]: Adjusting osd_memory_target on smithi156 to 8032M 2024-04-26T04:36:33.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:33.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:33.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:33.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:33.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:33.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:33.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:33.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:33.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[27570]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:33.671 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[39130]: Detected new or changed devices on smithi156 2024-04-26T04:36:33.671 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:33.671 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:33.671 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:33.671 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:33.671 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[39130]: Adjusting osd_memory_target on smithi156 to 8032M 2024-04-26T04:36:33.671 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:33.671 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:33.671 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:33.671 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:33.671 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:33.671 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:33.672 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:33.672 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:33.672 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:33 smithi156 ceph-mon[39130]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:33 smithi171 ceph-mon[31139]: Detected new or changed devices on smithi156 2024-04-26T04:36:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:33 smithi171 ceph-mon[31139]: Adjusting osd_memory_target on smithi156 to 8032M 2024-04-26T04:36:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:33.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:33.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:33.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:33.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:33.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:33 smithi171 ceph-mon[31139]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:34.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:34 smithi156 ceph-mon[27570]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:36:34.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:34 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:36:34.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:34 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:36:34.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:34 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:34.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:34 smithi156 ceph-mon[39130]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:36:34.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:34 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:36:34.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:34 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:36:34.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:34 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:35.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:34 smithi171 ceph-mon[31139]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:36:35.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:34 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:36:35.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:34 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:36:35.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:34 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:35.737 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:35 smithi156 ceph-mon[27570]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:35.737 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:35 smithi156 ceph-mon[39130]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:35.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:35 smithi171 ceph-mon[31139]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:37.244 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:36 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/91310267' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d0051bbe-c0cd-42f4-9d8c-6b0a3e71886d"}]: dispatch 2024-04-26T04:36:37.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:36 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/91310267' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d0051bbe-c0cd-42f4-9d8c-6b0a3e71886d"}]': finished 2024-04-26T04:36:37.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:36 smithi156 ceph-mon[27570]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T04:36:37.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:36 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:37.245 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:36 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/91310267' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d0051bbe-c0cd-42f4-9d8c-6b0a3e71886d"}]: dispatch 2024-04-26T04:36:37.245 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:36 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/91310267' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d0051bbe-c0cd-42f4-9d8c-6b0a3e71886d"}]': finished 2024-04-26T04:36:37.245 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:36 smithi156 ceph-mon[39130]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T04:36:37.245 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:36 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:37.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:36 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/91310267' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d0051bbe-c0cd-42f4-9d8c-6b0a3e71886d"}]: dispatch 2024-04-26T04:36:37.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:36 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/91310267' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d0051bbe-c0cd-42f4-9d8c-6b0a3e71886d"}]': finished 2024-04-26T04:36:37.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:36 smithi171 ceph-mon[31139]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T04:36:37.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:36 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:38.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:37 smithi171 ceph-mon[31139]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:38.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:37 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2995553364' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:36:38.256 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:37 smithi156 ceph-mon[39130]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:38.256 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:37 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2995553364' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:36:38.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:37 smithi156 ceph-mon[27570]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:38.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:37 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2995553364' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:36:40.248 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:39 smithi156 ceph-mon[39130]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:40.249 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:39 smithi156 ceph-mon[27570]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:40.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:39 smithi171 ceph-mon[31139]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:42.238 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:41 smithi156 ceph-mon[27570]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:42.238 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:41 smithi156 ceph-mon[39130]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:42.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:41 smithi171 ceph-mon[31139]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:44.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:43 smithi171 ceph-mon[31139]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:44.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:43 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:36:44.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:43 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:44.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:43 smithi171 ceph-mon[31139]: Deploying daemon osd.2 on smithi156 2024-04-26T04:36:44.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:43 smithi156 ceph-mon[39130]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:44.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:43 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:36:44.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:43 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:44.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:43 smithi156 ceph-mon[39130]: Deploying daemon osd.2 on smithi156 2024-04-26T04:36:44.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:43 smithi156 ceph-mon[27570]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:44.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:43 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:36:44.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:43 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:44.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:43 smithi156 ceph-mon[27570]: Deploying daemon osd.2 on smithi156 2024-04-26T04:36:46.149 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:45 smithi156 ceph-mon[27570]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:46.150 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:45 smithi156 ceph-mon[39130]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:46.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:45 smithi171 ceph-mon[31139]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:48.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:47 smithi156 ceph-mon[27570]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:48.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:47 smithi156 ceph-mon[39130]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:48.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:47 smithi171 ceph-mon[31139]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:49.708 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:49 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:49.708 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:49 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:49.708 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:49 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:49.709 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:49 smithi156 ceph-mon[39130]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:49.709 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:49 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:49.709 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:49 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:49.709 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:49 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:49.709 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:49 smithi156 ceph-mon[27570]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:49.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:49 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:49.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:49 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:49.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:49 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:49.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:49 smithi171 ceph-mon[31139]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:52.078 INFO:teuthology.orchestra.run.smithi156.stdout:Created osd(s) 2 on host 'smithi156' 2024-04-26T04:36:52.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[27570]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:52.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[27570]: from='osd.2 [v2:172.21.15.156:6818/3653838089,v1:172.21.15.156:6819/3653838089]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:36:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[27570]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:36:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:52.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[39130]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:52.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:52.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[39130]: from='osd.2 [v2:172.21.15.156:6818/3653838089,v1:172.21.15.156:6819/3653838089]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:36:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[39130]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:36:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:52 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:52.453 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:52 smithi171 ceph-mon[31139]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:52.453 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:52 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:52.453 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:52 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:52.453 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:52 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:52.453 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:52 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:52.453 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:52 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:52.453 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:52 smithi171 ceph-mon[31139]: from='osd.2 [v2:172.21.15.156:6818/3653838089,v1:172.21.15.156:6819/3653838089]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:36:52.453 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:52 smithi171 ceph-mon[31139]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:36:52.453 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:52 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:52.454 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:52 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:52.454 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:52 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:52.685 DEBUG:teuthology.orchestra.run.smithi156:osd.2> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.2.service 2024-04-26T04:36:52.727 INFO:tasks.cephadm:Deploying osd.3 on smithi156 with /dev/vg_nvme/lv_1... 2024-04-26T04:36:52.727 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-26T04:36:53.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:53 smithi171 ceph-mon[31139]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:36:53.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:53 smithi171 ceph-mon[31139]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T04:36:53.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:53 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:53.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:53 smithi171 ceph-mon[31139]: from='osd.2 [v2:172.21.15.156:6818/3653838089,v1:172.21.15.156:6819/3653838089]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:36:53.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:53 smithi171 ceph-mon[31139]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:36:53.545 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:53 smithi156 ceph-mon[27570]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:36:53.545 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:53 smithi156 ceph-mon[27570]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T04:36:53.546 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:53 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:53.546 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:53 smithi156 ceph-mon[27570]: from='osd.2 [v2:172.21.15.156:6818/3653838089,v1:172.21.15.156:6819/3653838089]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:36:53.546 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:53 smithi156 ceph-mon[27570]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:36:53.546 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:53 smithi156 ceph-mon[39130]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:36:53.546 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:53 smithi156 ceph-mon[39130]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T04:36:53.546 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:53 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:53.546 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:53 smithi156 ceph-mon[39130]: from='osd.2 [v2:172.21.15.156:6818/3653838089,v1:172.21.15.156:6819/3653838089]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:36:53.546 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:53 smithi156 ceph-mon[39130]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:36:53.547 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:36:53 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2[56200]: 2024-04-26T04:36:53.197+0000 7f008e8aa700 -1 osd.2 0 waiting for initial osdmap 2024-04-26T04:36:53.547 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:36:53 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2[56200]: 2024-04-26T04:36:53.200+0000 7f008a6a7700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:36:53.826 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:36:54.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:54 smithi156 ceph-mon[27570]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:54.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:54 smithi156 ceph-mon[27570]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-04-26T04:36:54.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:54 smithi156 ceph-mon[27570]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T04:36:54.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:54 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:54.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:54 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:54.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:54 smithi156 ceph-mon[39130]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:54.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:54 smithi156 ceph-mon[39130]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-04-26T04:36:54.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:54 smithi156 ceph-mon[39130]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T04:36:54.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:54 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:54.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:54 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:54.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:54 smithi171 ceph-mon[31139]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:54.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:54 smithi171 ceph-mon[31139]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-04-26T04:36:54.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:54 smithi171 ceph-mon[31139]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T04:36:54.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:54 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:54.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:54 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:55.388 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:55 smithi156 ceph-mon[39130]: purged_snaps scrub starts 2024-04-26T04:36:55.388 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:55 smithi156 ceph-mon[39130]: purged_snaps scrub ok 2024-04-26T04:36:55.389 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:55 smithi156 ceph-mon[39130]: osd.2 [v2:172.21.15.156:6818/3653838089,v1:172.21.15.156:6819/3653838089] boot 2024-04-26T04:36:55.389 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:55 smithi156 ceph-mon[39130]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T04:36:55.389 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:55 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:55.389 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:55 smithi156 ceph-mon[27570]: purged_snaps scrub starts 2024-04-26T04:36:55.389 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:55 smithi156 ceph-mon[27570]: purged_snaps scrub ok 2024-04-26T04:36:55.389 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:55 smithi156 ceph-mon[27570]: osd.2 [v2:172.21.15.156:6818/3653838089,v1:172.21.15.156:6819/3653838089] boot 2024-04-26T04:36:55.389 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:55 smithi156 ceph-mon[27570]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T04:36:55.389 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:55 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:55.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:55 smithi171 ceph-mon[31139]: purged_snaps scrub starts 2024-04-26T04:36:55.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:55 smithi171 ceph-mon[31139]: purged_snaps scrub ok 2024-04-26T04:36:55.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:55 smithi171 ceph-mon[31139]: osd.2 [v2:172.21.15.156:6818/3653838089,v1:172.21.15.156:6819/3653838089] boot 2024-04-26T04:36:55.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:55 smithi171 ceph-mon[31139]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T04:36:55.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:55 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:56.232 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-26T04:36:56.232 INFO:teuthology.orchestra.run.smithi156.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-26T04:36:56.232 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10+0 records in 2024-04-26T04:36:56.232 INFO:teuthology.orchestra.run.smithi156.stderr:10+0 records out 2024-04-26T04:36:56.232 INFO:teuthology.orchestra.run.smithi156.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0100944 s, 1.0 GB/s 2024-04-26T04:36:56.232 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping successful for: 2024-04-26T04:36:56.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:56 smithi171 ceph-mon[31139]: pgmap v74: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:56.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:56 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T04:36:56.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:56 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T04:36:56.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:56 smithi171 ceph-mon[31139]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T04:36:56.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:56 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:36:56.549 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:56 smithi156 ceph-mon[27570]: pgmap v74: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:56.549 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:56 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T04:36:56.549 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:56 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T04:36:56.550 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:56 smithi156 ceph-mon[27570]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T04:36:56.550 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:56 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:36:56.550 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:56 smithi156 ceph-mon[39130]: pgmap v74: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:56.550 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:56 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T04:36:56.550 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:56 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T04:36:56.550 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:56 smithi156 ceph-mon[39130]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T04:36:56.550 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:56 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:36:56.817 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:36:56 smithi156 sudo[59998]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:36:56.817 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:36:56 smithi156 sudo[59998]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:56.817 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:36:56 smithi156 sudo[59998]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:56.817 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:36:56 smithi156 sudo[59998]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:57.087 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:36:56 smithi156 sudo[60033]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:36:57.087 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:36:56 smithi156 sudo[60033]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:57.087 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:36:56 smithi156 sudo[60033]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:57.087 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:36:56 smithi156 sudo[60033]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:57.340 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:36:57 smithi156 sudo[60044]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:36:57.340 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:36:57 smithi156 sudo[60044]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:57.340 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:36:57 smithi156 sudo[60044]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:57.340 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:36:57 smithi156 sudo[60044]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:57.340 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:36:57.340 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[27570]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T04:36:57.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:57.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:57.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:57.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:57.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:57.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:57.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:57.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:57.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:57.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:57.342 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:36:57.342 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[39130]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T04:36:57.342 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:57.342 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:57.342 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:57.342 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:57.342 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:57.342 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:57.342 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:57.342 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:57.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:57.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:57 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:57.344 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi156:vg_nvme/lv_1 2024-04-26T04:36:57.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:57 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:36:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:57 smithi171 ceph-mon[31139]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T04:36:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:57 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:57 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:57 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:57 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:57 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:57 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:57 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:57 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:57 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:57 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:57.593 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:36:57 smithi156 sudo[60047]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:36:57.593 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:36:57 smithi156 sudo[60047]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:57.593 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:36:57 smithi156 sudo[60047]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:57.593 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:36:57 smithi156 sudo[60047]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:57.597 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:36:57.855 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:36:57 smithi156 sudo[60108]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:36:57.856 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:36:57 smithi156 sudo[60108]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:57.856 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:36:57 smithi156 sudo[60108]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:57.856 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:36:57 smithi156 sudo[60108]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:58.120 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:36:57 smithi156 sudo[60144]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:36:58.120 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:36:57 smithi156 sudo[60144]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:58.120 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:36:57 smithi156 sudo[60144]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:58.120 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:36:57 smithi156 sudo[60144]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:58.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:58 smithi156 sudo[60165]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:36:58.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:58 smithi156 sudo[60165]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:58.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:58 smithi156 sudo[60165]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:58.376 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:58 smithi156 sudo[60165]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:58.639 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:58 smithi171 ceph-mon[31139]: Detected new or changed devices on smithi156 2024-04-26T04:36:58.639 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:58 smithi171 ceph-mon[31139]: Adjusting osd_memory_target on smithi156 to 5354M 2024-04-26T04:36:58.639 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:58 smithi171 ceph-mon[31139]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:58.639 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:58 smithi171 ceph-mon[31139]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T04:36:58.639 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:58 smithi171 ceph-mon[31139]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T04:36:58.640 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:58 smithi171 ceph-mon[31139]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:36:58.640 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:58 smithi171 sudo[33973]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:36:58.640 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:58 smithi171 sudo[33973]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:58.640 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:58 smithi171 sudo[33973]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:58.640 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:58 smithi171 sudo[33973]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:58.749 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:58 smithi156 ceph-mon[39130]: Detected new or changed devices on smithi156 2024-04-26T04:36:58.749 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:58 smithi156 ceph-mon[39130]: Adjusting osd_memory_target on smithi156 to 5354M 2024-04-26T04:36:58.749 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:58 smithi156 ceph-mon[39130]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:58.750 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:58 smithi156 ceph-mon[39130]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T04:36:58.750 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:58 smithi156 ceph-mon[39130]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T04:36:58.750 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:58 smithi156 ceph-mon[39130]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:36:58.750 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:58 smithi156 sudo[60171]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:36:58.750 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:58 smithi156 sudo[60171]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:58.750 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:58 smithi156 sudo[60171]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:58.750 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:58 smithi156 sudo[60171]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:58.750 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:58 smithi156 ceph-mon[27570]: Detected new or changed devices on smithi156 2024-04-26T04:36:58.750 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:58 smithi156 ceph-mon[27570]: Adjusting osd_memory_target on smithi156 to 5354M 2024-04-26T04:36:58.750 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:58 smithi156 ceph-mon[27570]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:58.750 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:58 smithi156 ceph-mon[27570]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T04:36:58.750 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:58 smithi156 ceph-mon[27570]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T04:36:58.751 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:58 smithi156 ceph-mon[27570]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:36:58.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:58 smithi171 sudo[33976]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:36:59.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:58 smithi171 sudo[33976]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:59.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:58 smithi171 sudo[33976]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:59.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:58 smithi171 sudo[33976]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:59.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 sudo[60218]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:36:59.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 sudo[60218]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:59.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 sudo[60218]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:59.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 sudo[60218]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:59.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:59 smithi171 ceph-mon[31139]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:36:59.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:59.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:59.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:36:59.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:59 smithi171 ceph-mon[31139]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:36:59.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:59.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:59.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:36:59.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:59 smithi171 ceph-mon[31139]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:36:59.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:59.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:59.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:59.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:36:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:36:59.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 sudo[60250]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:36:59.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 sudo[60250]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:59.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 sudo[60250]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:59.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 sudo[60250]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:59.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[39130]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:36:59.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:59.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:59.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:36:59.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[39130]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:36:59.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:59.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:59.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:36:59.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[39130]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:36:59.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:59.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:59.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:59.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:36:59.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[27570]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:36:59.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:59.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:59.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:36:59.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[27570]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:36:59.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:59.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:59.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:36:59.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[27570]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:36:59.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:36:59.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:59.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:59.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:36:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:37:00.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:00 smithi171 ceph-mon[31139]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:37:00.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:00 smithi171 ceph-mon[31139]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:00.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:00 smithi171 ceph-mon[31139]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:37:00.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:00 smithi171 ceph-mon[31139]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:00.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:00 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:00.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:00 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:00.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:00 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:00.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:00 smithi156 ceph-mon[27570]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:37:00.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:00 smithi156 ceph-mon[27570]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:00.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:00 smithi156 ceph-mon[27570]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:37:00.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:00 smithi156 ceph-mon[27570]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:00.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:00 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:00.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:00 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:00.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:00 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:00.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:00 smithi156 ceph-mon[39130]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:37:00.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:00 smithi156 ceph-mon[39130]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:00.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:00 smithi156 ceph-mon[39130]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:37:00.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:00 smithi156 ceph-mon[39130]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:00.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:00 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:00.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:00 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:00.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:00 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:01.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:01 smithi156 ceph-mon[27570]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:01.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:01 smithi156 ceph-mon[39130]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:01.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:01 smithi171 ceph-mon[31139]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:02.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:02 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1798034672' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "479e109c-1e4a-4124-9442-f85d63df6d04"}]: dispatch 2024-04-26T04:37:02.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:02 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1798034672' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "479e109c-1e4a-4124-9442-f85d63df6d04"}]': finished 2024-04-26T04:37:02.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:02 smithi171 ceph-mon[31139]: osdmap e22: 4 total, 3 up, 4 in 2024-04-26T04:37:02.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:02 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:02.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:02 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1798034672' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "479e109c-1e4a-4124-9442-f85d63df6d04"}]: dispatch 2024-04-26T04:37:02.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:02 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1798034672' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "479e109c-1e4a-4124-9442-f85d63df6d04"}]': finished 2024-04-26T04:37:02.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:02 smithi156 ceph-mon[27570]: osdmap e22: 4 total, 3 up, 4 in 2024-04-26T04:37:02.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:02 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:02.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:02 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1798034672' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "479e109c-1e4a-4124-9442-f85d63df6d04"}]: dispatch 2024-04-26T04:37:02.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:02 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1798034672' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "479e109c-1e4a-4124-9442-f85d63df6d04"}]': finished 2024-04-26T04:37:02.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:02 smithi156 ceph-mon[39130]: osdmap e22: 4 total, 3 up, 4 in 2024-04-26T04:37:02.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:02 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:03.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:03 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3971625751' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:03.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:03 smithi171 ceph-mon[31139]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:03.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:03 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3971625751' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:03.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:03 smithi156 ceph-mon[39130]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:03.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:03 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3971625751' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:03.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:03 smithi156 ceph-mon[27570]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:06.170 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:05 smithi156 ceph-mon[27570]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:06.171 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:05 smithi156 ceph-mon[39130]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:06.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:05 smithi171 ceph-mon[31139]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:08.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:07 smithi156 ceph-mon[27570]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:08.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:07 smithi156 ceph-mon[39130]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:08.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:07 smithi171 ceph-mon[31139]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:09.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:08 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:37:09.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:08 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:09.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:08 smithi156 ceph-mon[27570]: Deploying daemon osd.3 on smithi156 2024-04-26T04:37:09.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:08 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:37:09.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:08 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:09.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:08 smithi156 ceph-mon[39130]: Deploying daemon osd.3 on smithi156 2024-04-26T04:37:09.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:08 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:37:09.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:08 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:09.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:08 smithi171 ceph-mon[31139]: Deploying daemon osd.3 on smithi156 2024-04-26T04:37:10.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:09 smithi156 ceph-mon[27570]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:10.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:09 smithi156 ceph-mon[39130]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:10.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:09 smithi171 ceph-mon[31139]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:12.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:11 smithi156 ceph-mon[27570]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:12.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:11 smithi156 ceph-mon[39130]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:12.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:11 smithi171 ceph-mon[31139]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:14.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:14 smithi156 ceph-mon[27570]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:14.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:14 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:14.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:14 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:14.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:14 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:14.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:14 smithi156 ceph-mon[39130]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:14.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:14 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:14.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:14 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:14.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:14 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:14.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:14 smithi171 ceph-mon[31139]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:14.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:14 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:14.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:14 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:14.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:14 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:16.340 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:16 smithi156 ceph-mon[27570]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:16.340 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:16 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:16.340 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:16 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:16.341 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:16 smithi156 ceph-mon[39130]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:16.341 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:16 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:16.341 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:16 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:16.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:16 smithi171 ceph-mon[31139]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:16.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:16 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:16.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:16 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:16.989 INFO:teuthology.orchestra.run.smithi156.stdout:Created osd(s) 3 on host 'smithi156' 2024-04-26T04:37:17.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:17 smithi156 ceph-mon[27570]: from='osd.3 [v2:172.21.15.156:6826/2175305841,v1:172.21.15.156:6827/2175305841]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:37:17.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:17 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:17.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:17 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:17.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:17 smithi156 ceph-mon[39130]: from='osd.3 [v2:172.21.15.156:6826/2175305841,v1:172.21.15.156:6827/2175305841]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:37:17.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:17 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:17.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:17 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:17.630 DEBUG:teuthology.orchestra.run.smithi156:osd.3> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.3.service 2024-04-26T04:37:17.632 INFO:tasks.cephadm:Deploying osd.4 on smithi171 with /dev/vg_nvme/lv_4... 2024-04-26T04:37:17.632 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-26T04:37:17.659 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:17 smithi171 ceph-mon[31139]: from='osd.3 [v2:172.21.15.156:6826/2175305841,v1:172.21.15.156:6827/2175305841]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:37:17.660 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:17 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:17.660 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:17 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:17.889 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:37:18.509 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:18 smithi156 ceph-mon[27570]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:18.509 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:18 smithi156 ceph-mon[27570]: from='osd.3 [v2:172.21.15.156:6826/2175305841,v1:172.21.15.156:6827/2175305841]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:37:18.509 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:18 smithi156 ceph-mon[27570]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T04:37:18.509 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:18 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:18.509 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:18 smithi156 ceph-mon[27570]: from='osd.3 [v2:172.21.15.156:6826/2175305841,v1:172.21.15.156:6827/2175305841]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:37:18.509 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:18 smithi156 ceph-mon[39130]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:18.509 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:18 smithi156 ceph-mon[39130]: from='osd.3 [v2:172.21.15.156:6826/2175305841,v1:172.21.15.156:6827/2175305841]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:37:18.509 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:18 smithi156 ceph-mon[39130]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T04:37:18.509 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:18 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:18.510 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:18 smithi156 ceph-mon[39130]: from='osd.3 [v2:172.21.15.156:6826/2175305841,v1:172.21.15.156:6827/2175305841]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:37:18.510 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:37:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3[62345]: 2024-04-26T04:37:18.333+0000 7f74a054c700 -1 osd.3 0 waiting for initial osdmap 2024-04-26T04:37:18.510 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:37:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3[62345]: 2024-04-26T04:37:18.339+0000 7f7497b40700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:37:18.693 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:18 smithi171 ceph-mon[31139]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:18.694 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:18 smithi171 ceph-mon[31139]: from='osd.3 [v2:172.21.15.156:6826/2175305841,v1:172.21.15.156:6827/2175305841]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:37:18.694 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:18 smithi171 ceph-mon[31139]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T04:37:18.694 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:18 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:18.694 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:18 smithi171 ceph-mon[31139]: from='osd.3 [v2:172.21.15.156:6826/2175305841,v1:172.21.15.156:6827/2175305841]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:37:19.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:19 smithi171 ceph-mon[31139]: from='osd.3 [v2:172.21.15.156:6826/2175305841,v1:172.21.15.156:6827/2175305841]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-04-26T04:37:19.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:19 smithi171 ceph-mon[31139]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T04:37:19.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:19 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:19.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:19 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:19.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:19 smithi156 ceph-mon[27570]: from='osd.3 [v2:172.21.15.156:6826/2175305841,v1:172.21.15.156:6827/2175305841]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-04-26T04:37:19.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:19 smithi156 ceph-mon[27570]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T04:37:19.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:19 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:19.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:19 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:19.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:19 smithi156 ceph-mon[39130]: from='osd.3 [v2:172.21.15.156:6826/2175305841,v1:172.21.15.156:6827/2175305841]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-04-26T04:37:19.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:19 smithi156 ceph-mon[39130]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T04:37:19.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:19 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:19.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:19 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:19.765 INFO:teuthology.orchestra.run.smithi171.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-26T04:37:19.765 INFO:teuthology.orchestra.run.smithi171.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-26T04:37:19.765 INFO:teuthology.orchestra.run.smithi171.stderr: stderr: 10+0 records in 2024-04-26T04:37:19.766 INFO:teuthology.orchestra.run.smithi171.stderr:10+0 records out 2024-04-26T04:37:19.766 INFO:teuthology.orchestra.run.smithi171.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.011595 s, 904 MB/s 2024-04-26T04:37:19.766 INFO:teuthology.orchestra.run.smithi171.stderr:--> Zapping successful for: 2024-04-26T04:37:20.410 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi171:vg_nvme/lv_4 2024-04-26T04:37:20.696 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:37:20.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:20 smithi171 ceph-mon[31139]: purged_snaps scrub starts 2024-04-26T04:37:20.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:20 smithi171 ceph-mon[31139]: purged_snaps scrub ok 2024-04-26T04:37:20.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:20 smithi171 ceph-mon[31139]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:20.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:20 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:20.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:20 smithi171 ceph-mon[31139]: osd.3 [v2:172.21.15.156:6826/2175305841,v1:172.21.15.156:6827/2175305841] boot 2024-04-26T04:37:20.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:20 smithi171 ceph-mon[31139]: osdmap e25: 4 total, 4 up, 4 in 2024-04-26T04:37:20.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:20 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:20.771 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:20 smithi156 ceph-mon[27570]: purged_snaps scrub starts 2024-04-26T04:37:20.771 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:20 smithi156 ceph-mon[27570]: purged_snaps scrub ok 2024-04-26T04:37:20.771 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:20 smithi156 ceph-mon[27570]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:20.771 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:20 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:20.771 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:20 smithi156 ceph-mon[27570]: osd.3 [v2:172.21.15.156:6826/2175305841,v1:172.21.15.156:6827/2175305841] boot 2024-04-26T04:37:20.771 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:20 smithi156 ceph-mon[27570]: osdmap e25: 4 total, 4 up, 4 in 2024-04-26T04:37:20.771 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:20 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:20.772 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:20 smithi156 ceph-mon[39130]: purged_snaps scrub starts 2024-04-26T04:37:20.772 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:20 smithi156 ceph-mon[39130]: purged_snaps scrub ok 2024-04-26T04:37:20.772 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:20 smithi156 ceph-mon[39130]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:37:20.772 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:20 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:20.772 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:20 smithi156 ceph-mon[39130]: osd.3 [v2:172.21.15.156:6826/2175305841,v1:172.21.15.156:6827/2175305841] boot 2024-04-26T04:37:20.772 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:20 smithi156 ceph-mon[39130]: osdmap e25: 4 total, 4 up, 4 in 2024-04-26T04:37:20.772 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:20 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:37:21.502 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:21 smithi171 ceph-mon[31139]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T04:37:21.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:21.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:21.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:21.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:21.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:21.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:21.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:21.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:21.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:21.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:21.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:21.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:21.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:21.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:21.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[27570]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T04:37:21.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:21.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:21.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:21.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:21.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:21.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:21.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:21.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:21.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:21.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:21.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:21.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:21.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:21.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:21.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[39130]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T04:37:21.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:21.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:21.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:21.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:21.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:21.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:21.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:21.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:21.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:21.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:21.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:21.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:21.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:21.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:21 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:22.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:22 smithi171 ceph-mon[31139]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:22.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:22 smithi171 ceph-mon[31139]: Detected new or changed devices on smithi156 2024-04-26T04:37:22.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:22 smithi171 ceph-mon[31139]: Adjusting osd_memory_target on smithi156 to 4016M 2024-04-26T04:37:22.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:22 smithi156 ceph-mon[27570]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:22.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:22 smithi156 ceph-mon[27570]: Detected new or changed devices on smithi156 2024-04-26T04:37:22.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:22 smithi156 ceph-mon[27570]: Adjusting osd_memory_target on smithi156 to 4016M 2024-04-26T04:37:22.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:22 smithi156 ceph-mon[39130]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:22.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:22 smithi156 ceph-mon[39130]: Detected new or changed devices on smithi156 2024-04-26T04:37:22.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:22 smithi156 ceph-mon[39130]: Adjusting osd_memory_target on smithi156 to 4016M 2024-04-26T04:37:23.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:23 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:23.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:23 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:23.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:23 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:23.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:23 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:23.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:23 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:23.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:23 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:23.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:23 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:23.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:23 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:23.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:23 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:24.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:24 smithi171 ceph-mon[31139]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:24.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:24 smithi171 ceph-mon[31139]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:24.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:24 smithi156 ceph-mon[27570]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:24.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:24 smithi156 ceph-mon[27570]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:24.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:24 smithi156 ceph-mon[39130]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:24.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:24 smithi156 ceph-mon[39130]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:25.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:25 smithi171 ceph-mon[31139]: from='client.? 172.21.15.171:0/2391206621' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b090dbe4-ed23-4015-a864-074aa64cace4"}]: dispatch 2024-04-26T04:37:25.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:25 smithi171 ceph-mon[31139]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b090dbe4-ed23-4015-a864-074aa64cace4"}]: dispatch 2024-04-26T04:37:25.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:25 smithi171 ceph-mon[31139]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b090dbe4-ed23-4015-a864-074aa64cace4"}]': finished 2024-04-26T04:37:25.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:25 smithi171 ceph-mon[31139]: osdmap e27: 5 total, 4 up, 5 in 2024-04-26T04:37:25.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:25 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:37:25.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:25 smithi156 ceph-mon[27570]: from='client.? 172.21.15.171:0/2391206621' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b090dbe4-ed23-4015-a864-074aa64cace4"}]: dispatch 2024-04-26T04:37:25.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:25 smithi156 ceph-mon[27570]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b090dbe4-ed23-4015-a864-074aa64cace4"}]: dispatch 2024-04-26T04:37:25.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:25 smithi156 ceph-mon[27570]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b090dbe4-ed23-4015-a864-074aa64cace4"}]': finished 2024-04-26T04:37:25.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:25 smithi156 ceph-mon[27570]: osdmap e27: 5 total, 4 up, 5 in 2024-04-26T04:37:25.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:25 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:37:25.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:25 smithi156 ceph-mon[39130]: from='client.? 172.21.15.171:0/2391206621' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b090dbe4-ed23-4015-a864-074aa64cace4"}]: dispatch 2024-04-26T04:37:25.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:25 smithi156 ceph-mon[39130]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b090dbe4-ed23-4015-a864-074aa64cace4"}]: dispatch 2024-04-26T04:37:25.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:25 smithi156 ceph-mon[39130]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b090dbe4-ed23-4015-a864-074aa64cace4"}]': finished 2024-04-26T04:37:25.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:25 smithi156 ceph-mon[39130]: osdmap e27: 5 total, 4 up, 5 in 2024-04-26T04:37:25.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:25 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:37:26.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:26 smithi171 ceph-mon[31139]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:26.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:26 smithi171 ceph-mon[31139]: from='client.? 172.21.15.171:0/2154311895' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:26.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:26 smithi156 ceph-mon[27570]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:26.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:26 smithi156 ceph-mon[27570]: from='client.? 172.21.15.171:0/2154311895' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:26.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:26 smithi156 ceph-mon[39130]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:26.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:26 smithi156 ceph-mon[39130]: from='client.? 172.21.15.171:0/2154311895' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:28.641 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:28 smithi171 ceph-mon[31139]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:28.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:28 smithi156 ceph-mon[39130]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:28.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:28 smithi156 ceph-mon[27570]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:30.723 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:30 smithi171 ceph-mon[31139]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:30.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:30 smithi156 ceph-mon[39130]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:30.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:30 smithi156 ceph-mon[27570]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:31.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:31 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:37:31.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:31 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:31.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:31 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:37:31.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:31 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:31.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:31 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:37:31.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:31 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:32.734 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:32 smithi171 ceph-mon[31139]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:32.734 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:32 smithi171 ceph-mon[31139]: Deploying daemon osd.4 on smithi171 2024-04-26T04:37:32.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:32 smithi156 ceph-mon[39130]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:32.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:32 smithi156 ceph-mon[39130]: Deploying daemon osd.4 on smithi171 2024-04-26T04:37:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:32 smithi156 ceph-mon[27570]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:32 smithi156 ceph-mon[27570]: Deploying daemon osd.4 on smithi171 2024-04-26T04:37:34.692 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:34 smithi171 ceph-mon[31139]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:34.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:34 smithi156 ceph-mon[27570]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:34.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:34 smithi156 ceph-mon[39130]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:36.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:36 smithi171 ceph-mon[31139]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:36.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:36 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:36.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:36 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:36.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:36 smithi156 ceph-mon[27570]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:36.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:36 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:36.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:36 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:36.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:36 smithi156 ceph-mon[39130]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:36.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:36 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:36 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:37.657 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:37 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:37.744 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:37 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:37.745 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:37 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:38.721 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:38 smithi171 ceph-mon[31139]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:38.764 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:38 smithi156 ceph-mon[27570]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:38.764 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:38 smithi156 ceph-mon[39130]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:39.762 INFO:teuthology.orchestra.run.smithi171.stdout:Created osd(s) 4 on host 'smithi171' 2024-04-26T04:37:39.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:39 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:40.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:39 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:40.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:39 smithi171 ceph-mon[31139]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:40.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:39 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:40.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:39 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:40.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:39 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:40.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:39 smithi171 ceph-mon[31139]: from='osd.4 [v2:172.21.15.171:6800/2255415442,v1:172.21.15.171:6801/2255415442]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:37:40.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:39 smithi171 ceph-mon[31139]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:37:40.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:39 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:40.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:39 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:40.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:39 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:40.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:40.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:40.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[27570]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:40.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:40.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:40.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:40.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[27570]: from='osd.4 [v2:172.21.15.171:6800/2255415442,v1:172.21.15.171:6801/2255415442]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:37:40.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[27570]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:37:40.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:40.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:40.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:40.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:40.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:40.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[39130]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:40.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:40.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:40.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:40.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[39130]: from='osd.4 [v2:172.21.15.171:6800/2255415442,v1:172.21.15.171:6801/2255415442]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:37:40.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[39130]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:37:40.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:40.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:40.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:39 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:40.420 DEBUG:teuthology.orchestra.run.smithi171:osd.4> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.4.service 2024-04-26T04:37:40.422 INFO:tasks.cephadm:Deploying osd.5 on smithi171 with /dev/vg_nvme/lv_3... 2024-04-26T04:37:40.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-26T04:37:41.129 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:37:41.552 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:41 smithi171 ceph-mon[31139]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:37:41.552 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:41 smithi171 ceph-mon[31139]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T04:37:41.552 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:41 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:37:41.552 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:41 smithi171 ceph-mon[31139]: from='osd.4 [v2:172.21.15.171:6800/2255415442,v1:172.21.15.171:6801/2255415442]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:37:41.553 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:41 smithi171 ceph-mon[31139]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:37:41.553 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:41 smithi171 ceph-mon[31139]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:41.553 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:37:41 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[36606]: 2024-04-26T04:37:41.448+0000 7efe63dce700 -1 osd.4 0 waiting for initial osdmap 2024-04-26T04:37:41.553 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:37:41 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[36606]: 2024-04-26T04:37:41.452+0000 7efe5ebc9700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:37:41.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:41 smithi156 ceph-mon[39130]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:37:41.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:41 smithi156 ceph-mon[39130]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T04:37:41.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:41 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:37:41.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:41 smithi156 ceph-mon[39130]: from='osd.4 [v2:172.21.15.171:6800/2255415442,v1:172.21.15.171:6801/2255415442]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:37:41.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:41 smithi156 ceph-mon[39130]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:37:41.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:41 smithi156 ceph-mon[39130]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:41.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:41 smithi156 ceph-mon[27570]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:37:41.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:41 smithi156 ceph-mon[27570]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T04:37:41.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:41 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:37:41.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:41 smithi156 ceph-mon[27570]: from='osd.4 [v2:172.21.15.171:6800/2255415442,v1:172.21.15.171:6801/2255415442]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:37:41.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:41 smithi156 ceph-mon[27570]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:37:41.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:41 smithi156 ceph-mon[27570]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:37:42.715 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:42 smithi171 ceph-mon[31139]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-04-26T04:37:42.715 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:42 smithi171 ceph-mon[31139]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T04:37:42.716 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:42 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:37:42.716 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:42 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:37:42.716 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:42 smithi171 ceph-mon[31139]: osd.4 [v2:172.21.15.171:6800/2255415442,v1:172.21.15.171:6801/2255415442] boot 2024-04-26T04:37:42.716 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:42 smithi171 ceph-mon[31139]: osdmap e30: 5 total, 5 up, 5 in 2024-04-26T04:37:42.716 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:42 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:37:42.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:42 smithi156 ceph-mon[27570]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-04-26T04:37:42.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:42 smithi156 ceph-mon[27570]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T04:37:42.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:37:42.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:37:42.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:42 smithi156 ceph-mon[27570]: osd.4 [v2:172.21.15.171:6800/2255415442,v1:172.21.15.171:6801/2255415442] boot 2024-04-26T04:37:42.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:42 smithi156 ceph-mon[27570]: osdmap e30: 5 total, 5 up, 5 in 2024-04-26T04:37:42.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:42 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:37:42.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:42 smithi156 ceph-mon[39130]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-04-26T04:37:42.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:42 smithi156 ceph-mon[39130]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T04:37:42.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:42 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:37:42.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:42 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:37:42.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:42 smithi156 ceph-mon[39130]: osd.4 [v2:172.21.15.171:6800/2255415442,v1:172.21.15.171:6801/2255415442] boot 2024-04-26T04:37:42.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:42 smithi156 ceph-mon[39130]: osdmap e30: 5 total, 5 up, 5 in 2024-04-26T04:37:42.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:42 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:37:43.748 INFO:teuthology.orchestra.run.smithi171.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-26T04:37:43.748 INFO:teuthology.orchestra.run.smithi171.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-26T04:37:43.748 INFO:teuthology.orchestra.run.smithi171.stderr: stderr: 10+0 records in 2024-04-26T04:37:43.748 INFO:teuthology.orchestra.run.smithi171.stderr:10+0 records out 2024-04-26T04:37:43.749 INFO:teuthology.orchestra.run.smithi171.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0159637 s, 657 MB/s 2024-04-26T04:37:43.749 INFO:teuthology.orchestra.run.smithi171.stderr:--> Zapping successful for: 2024-04-26T04:37:43.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:43 smithi171 ceph-mon[31139]: purged_snaps scrub starts 2024-04-26T04:37:43.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:43 smithi171 ceph-mon[31139]: purged_snaps scrub ok 2024-04-26T04:37:43.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:43 smithi171 ceph-mon[31139]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T04:37:43.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:43 smithi171 ceph-mon[31139]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:43.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:43 smithi156 ceph-mon[27570]: purged_snaps scrub starts 2024-04-26T04:37:43.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:43 smithi156 ceph-mon[27570]: purged_snaps scrub ok 2024-04-26T04:37:43.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:43 smithi156 ceph-mon[27570]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T04:37:43.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:43 smithi156 ceph-mon[27570]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:43.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:43 smithi156 ceph-mon[39130]: purged_snaps scrub starts 2024-04-26T04:37:43.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:43 smithi156 ceph-mon[39130]: purged_snaps scrub ok 2024-04-26T04:37:43.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:43 smithi156 ceph-mon[39130]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T04:37:43.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:43 smithi156 ceph-mon[39130]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:44.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:44 smithi171 ceph-mon[31139]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T04:37:44.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:44 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:44.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:44 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:44.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:44 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:44.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:44 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:44.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:44 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:44.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:44 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:44.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:44 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:44.927 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi171:vg_nvme/lv_3 2024-04-26T04:37:45.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:44 smithi156 ceph-mon[39130]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T04:37:45.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:44 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:45.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:44 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:45.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:44 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:45.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:44 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:45.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:44 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:45.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:44 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:45.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:44 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:45.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:44 smithi156 ceph-mon[27570]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T04:37:45.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:44 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:45.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:44 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:45.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:44 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:45.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:44 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:45.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:44 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:45.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:44 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:45.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:44 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:37:45.188 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:37:45.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:45 smithi171 ceph-mon[31139]: Detected new or changed devices on smithi171 2024-04-26T04:37:45.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:45 smithi171 ceph-mon[31139]: Adjusting osd_memory_target on smithi171 to 17088M 2024-04-26T04:37:46.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:45 smithi171 ceph-mon[31139]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T04:37:46.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:45 smithi171 ceph-mon[31139]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:46.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:45 smithi156 ceph-mon[39130]: Detected new or changed devices on smithi171 2024-04-26T04:37:46.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:45 smithi156 ceph-mon[39130]: Adjusting osd_memory_target on smithi171 to 17088M 2024-04-26T04:37:46.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:45 smithi156 ceph-mon[39130]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T04:37:46.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:45 smithi156 ceph-mon[39130]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:46.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:45 smithi156 ceph-mon[27570]: Detected new or changed devices on smithi171 2024-04-26T04:37:46.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:45 smithi156 ceph-mon[27570]: Adjusting osd_memory_target on smithi171 to 17088M 2024-04-26T04:37:46.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:45 smithi156 ceph-mon[27570]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T04:37:46.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:45 smithi156 ceph-mon[27570]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:46.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:46 smithi171 ceph-mon[31139]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T04:37:47.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:46 smithi156 ceph-mon[27570]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T04:37:47.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:46 smithi156 ceph-mon[39130]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T04:37:47.704 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:47 smithi171 ceph-mon[31139]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 133 KiB/s, 0 objects/s recovering 2024-04-26T04:37:47.705 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:47 smithi171 ceph-mon[31139]: from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:47.705 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:47 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:47.705 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:47 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:47.705 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:47 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:48.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:47 smithi156 ceph-mon[27570]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 133 KiB/s, 0 objects/s recovering 2024-04-26T04:37:48.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:47 smithi156 ceph-mon[27570]: from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:48.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:47 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:48.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:47 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:48.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:47 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:48.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:47 smithi156 ceph-mon[39130]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 133 KiB/s, 0 objects/s recovering 2024-04-26T04:37:48.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:47 smithi156 ceph-mon[39130]: from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:48.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:47 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:48.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:47 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:48.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:47 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:49 smithi171 ceph-mon[31139]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:37:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:49 smithi171 ceph-mon[31139]: from='client.? 172.21.15.171:0/3615516795' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f506481f-428b-40d6-a54b-8ef411672b53"}]: dispatch 2024-04-26T04:37:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:49 smithi171 ceph-mon[31139]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f506481f-428b-40d6-a54b-8ef411672b53"}]: dispatch 2024-04-26T04:37:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:49 smithi171 ceph-mon[31139]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f506481f-428b-40d6-a54b-8ef411672b53"}]': finished 2024-04-26T04:37:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:49 smithi171 ceph-mon[31139]: osdmap e35: 6 total, 5 up, 6 in 2024-04-26T04:37:50.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:49 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:50.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:49 smithi156 ceph-mon[39130]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:37:50.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:49 smithi156 ceph-mon[39130]: from='client.? 172.21.15.171:0/3615516795' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f506481f-428b-40d6-a54b-8ef411672b53"}]: dispatch 2024-04-26T04:37:50.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:49 smithi156 ceph-mon[39130]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f506481f-428b-40d6-a54b-8ef411672b53"}]: dispatch 2024-04-26T04:37:50.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:49 smithi156 ceph-mon[39130]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f506481f-428b-40d6-a54b-8ef411672b53"}]': finished 2024-04-26T04:37:50.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:49 smithi156 ceph-mon[39130]: osdmap e35: 6 total, 5 up, 6 in 2024-04-26T04:37:50.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:49 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:50.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:49 smithi156 ceph-mon[27570]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:37:50.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:49 smithi156 ceph-mon[27570]: from='client.? 172.21.15.171:0/3615516795' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f506481f-428b-40d6-a54b-8ef411672b53"}]: dispatch 2024-04-26T04:37:50.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:49 smithi156 ceph-mon[27570]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f506481f-428b-40d6-a54b-8ef411672b53"}]: dispatch 2024-04-26T04:37:50.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:49 smithi156 ceph-mon[27570]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f506481f-428b-40d6-a54b-8ef411672b53"}]': finished 2024-04-26T04:37:50.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:49 smithi156 ceph-mon[27570]: osdmap e35: 6 total, 5 up, 6 in 2024-04-26T04:37:50.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:49 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:51.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:50 smithi156 ceph-mon[27570]: from='client.? 172.21.15.171:0/4150345' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:51.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:51 smithi156 ceph-mon[39130]: from='client.? 172.21.15.171:0/4150345' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:51.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:51 smithi171 ceph-mon[31139]: from='client.? 172.21.15.171:0/4150345' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:52.338 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:52 smithi171 ceph-mon[31139]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-26T04:37:52.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:52 smithi156 ceph-mon[27570]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-26T04:37:52.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:52 smithi156 ceph-mon[39130]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-26T04:37:54.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:54 smithi156 ceph-mon[27570]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:37:54.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:54 smithi156 ceph-mon[39130]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:37:54.497 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:54 smithi171 ceph-mon[31139]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:37:56.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:56 smithi171 ceph-mon[31139]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-26T04:37:56.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:56 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:37:56.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:56 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:56.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:56 smithi156 ceph-mon[27570]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-26T04:37:56.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:56 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:37:56.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:56 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:56.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:56 smithi156 ceph-mon[39130]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-26T04:37:56.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:56 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:37:56.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:56 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:57.335 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:57 smithi171 ceph-mon[31139]: Deploying daemon osd.5 on smithi171 2024-04-26T04:37:57.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:57 smithi156 ceph-mon[27570]: Deploying daemon osd.5 on smithi171 2024-04-26T04:37:57.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:57 smithi156 ceph-mon[39130]: Deploying daemon osd.5 on smithi171 2024-04-26T04:37:58.323 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:37:58 smithi171 ceph-mon[31139]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:58.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:37:58 smithi156 ceph-mon[27570]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:58.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:37:58 smithi156 ceph-mon[39130]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:38:00.189 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:00 smithi171 ceph-mon[31139]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:38:00.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:00 smithi156 ceph-mon[27570]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:38:00.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:00 smithi156 ceph-mon[39130]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:38:02.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:02 smithi156 ceph-mon[27570]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:38:02.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:02 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:02.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:02 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:02.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:02 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:02.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:02 smithi156 ceph-mon[39130]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:38:02.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:02 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:02.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:02 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:02.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:02 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:02.372 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:02 smithi171 ceph-mon[31139]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:38:02.373 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:02 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:02.373 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:02 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:02.373 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:02 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:04.105 INFO:teuthology.orchestra.run.smithi171.stdout:Created osd(s) 5 on host 'smithi171' 2024-04-26T04:38:04.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:04 smithi156 ceph-mon[27570]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:38:04.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:04 smithi156 ceph-mon[39130]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:38:04.419 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:04 smithi171 ceph-mon[31139]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:38:04.714 DEBUG:teuthology.orchestra.run.smithi171:osd.5> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.5.service 2024-04-26T04:38:04.716 INFO:tasks.cephadm:Deploying osd.6 on smithi171 with /dev/vg_nvme/lv_2... 2024-04-26T04:38:04.717 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-26T04:38:05.022 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:38:05.314 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:05 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:05.315 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:05 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:05.315 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:05 smithi171 ceph-mon[31139]: from='osd.5 [v2:172.21.15.171:6808/2373798416,v1:172.21.15.171:6809/2373798416]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:38:05.315 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:05 smithi171 ceph-mon[31139]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:38:05.315 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:05 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:05.315 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:05 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:05.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:05 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:05.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:05 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:05.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:05 smithi156 ceph-mon[27570]: from='osd.5 [v2:172.21.15.171:6808/2373798416,v1:172.21.15.171:6809/2373798416]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:38:05.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:05 smithi156 ceph-mon[27570]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:38:05.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:05 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:05.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:05 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:05.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:05 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:05.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:05 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:05.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:05 smithi156 ceph-mon[39130]: from='osd.5 [v2:172.21.15.171:6808/2373798416,v1:172.21.15.171:6809/2373798416]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:38:05.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:05 smithi156 ceph-mon[39130]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:38:05.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:05 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:05.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:05 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:06 smithi171 ceph-mon[31139]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:38:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:06 smithi171 ceph-mon[31139]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T04:38:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:06 smithi171 ceph-mon[31139]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T04:38:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:06 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:38:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:06 smithi171 ceph-mon[31139]: from='osd.5 [v2:172.21.15.171:6808/2373798416,v1:172.21.15.171:6809/2373798416]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:06 smithi171 ceph-mon[31139]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:06.250 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:38:06 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:38:06.073+0000 7f8b63a48700 -1 osd.5 0 waiting for initial osdmap 2024-04-26T04:38:06.250 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:38:06 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:38:06.078+0000 7f8b5d040700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:38:06.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:06 smithi156 ceph-mon[27570]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:38:06.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:06 smithi156 ceph-mon[27570]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T04:38:06.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:06 smithi156 ceph-mon[27570]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T04:38:06.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:06 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:38:06.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:06 smithi156 ceph-mon[27570]: from='osd.5 [v2:172.21.15.171:6808/2373798416,v1:172.21.15.171:6809/2373798416]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:06.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:06 smithi156 ceph-mon[27570]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:06 smithi156 ceph-mon[39130]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:38:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:06 smithi156 ceph-mon[39130]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T04:38:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:06 smithi156 ceph-mon[39130]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T04:38:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:06 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:38:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:06 smithi156 ceph-mon[39130]: from='osd.5 [v2:172.21.15.171:6808/2373798416,v1:172.21.15.171:6809/2373798416]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:06 smithi156 ceph-mon[39130]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:07.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:07 smithi156 ceph-mon[27570]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-04-26T04:38:07.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:07 smithi156 ceph-mon[27570]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T04:38:07.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:07 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:38:07.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:07 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:38:07.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:07 smithi156 ceph-mon[27570]: osd.5 [v2:172.21.15.171:6808/2373798416,v1:172.21.15.171:6809/2373798416] boot 2024-04-26T04:38:07.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:07 smithi156 ceph-mon[27570]: osdmap e38: 6 total, 6 up, 6 in 2024-04-26T04:38:07.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:07 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:38:07.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:07 smithi156 ceph-mon[39130]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-04-26T04:38:07.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:07 smithi156 ceph-mon[39130]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T04:38:07.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:07 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:38:07.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:07 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:38:07.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:07 smithi156 ceph-mon[39130]: osd.5 [v2:172.21.15.171:6808/2373798416,v1:172.21.15.171:6809/2373798416] boot 2024-04-26T04:38:07.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:07 smithi156 ceph-mon[39130]: osdmap e38: 6 total, 6 up, 6 in 2024-04-26T04:38:07.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:07 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:38:07.484 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:07 smithi171 ceph-mon[31139]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-04-26T04:38:07.484 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:07 smithi171 ceph-mon[31139]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T04:38:07.484 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:07 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:38:07.484 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:07 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:38:07.484 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:07 smithi171 ceph-mon[31139]: osd.5 [v2:172.21.15.171:6808/2373798416,v1:172.21.15.171:6809/2373798416] boot 2024-04-26T04:38:07.484 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:07 smithi171 ceph-mon[31139]: osdmap e38: 6 total, 6 up, 6 in 2024-04-26T04:38:07.484 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:07 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:38:07.978 INFO:teuthology.orchestra.run.smithi171.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-26T04:38:07.978 INFO:teuthology.orchestra.run.smithi171.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-26T04:38:07.978 INFO:teuthology.orchestra.run.smithi171.stderr: stderr: 10+0 records in 2024-04-26T04:38:07.978 INFO:teuthology.orchestra.run.smithi171.stderr:10+0 records out 2024-04-26T04:38:07.978 INFO:teuthology.orchestra.run.smithi171.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0122553 s, 856 MB/s 2024-04-26T04:38:07.978 INFO:teuthology.orchestra.run.smithi171.stderr:--> Zapping successful for: 2024-04-26T04:38:08.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:08 smithi171 ceph-mon[31139]: purged_snaps scrub starts 2024-04-26T04:38:08.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:08 smithi171 ceph-mon[31139]: purged_snaps scrub ok 2024-04-26T04:38:08.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:08 smithi171 ceph-mon[31139]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:38:08.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:08 smithi171 ceph-mon[31139]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T04:38:08.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:08 smithi171 ceph-mon[31139]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T04:38:08.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:08 smithi156 ceph-mon[39130]: purged_snaps scrub starts 2024-04-26T04:38:08.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:08 smithi156 ceph-mon[39130]: purged_snaps scrub ok 2024-04-26T04:38:08.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:08 smithi156 ceph-mon[39130]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:38:08.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:08 smithi156 ceph-mon[39130]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T04:38:08.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:08 smithi156 ceph-mon[39130]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T04:38:08.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:08 smithi156 ceph-mon[27570]: purged_snaps scrub starts 2024-04-26T04:38:08.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:08 smithi156 ceph-mon[27570]: purged_snaps scrub ok 2024-04-26T04:38:08.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:08 smithi156 ceph-mon[27570]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:38:08.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:08 smithi156 ceph-mon[27570]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T04:38:08.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:08 smithi156 ceph-mon[27570]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T04:38:08.540 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi171:vg_nvme/lv_2 2024-04-26T04:38:08.873 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:38:09.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:09 smithi171 ceph-mon[31139]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T04:38:09.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:09 smithi171 ceph-mon[31139]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:10.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:09 smithi156 ceph-mon[27570]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T04:38:10.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:09 smithi156 ceph-mon[27570]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:10.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:09 smithi156 ceph-mon[39130]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T04:38:10.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:09 smithi156 ceph-mon[39130]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:10.936 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:10 smithi171 ceph-mon[31139]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T04:38:11.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:10 smithi156 ceph-mon[27570]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T04:38:11.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:10 smithi156 ceph-mon[39130]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T04:38:11.889 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T04:38:11.889 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: Detected new or changed devices on smithi171 2024-04-26T04:38:11.889 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:11.889 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:11.889 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:11.889 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:11.890 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: Adjusting osd_memory_target on smithi171 to 8544M 2024-04-26T04:38:11.890 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:11.890 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:11.890 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:11.890 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:11.890 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:11.890 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:11.890 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:11.890 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:11.890 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: pgmap v136: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:11.890 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:38:11.890 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:38:11.890 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:38:11.890 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:11 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:12.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T04:38:12.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: Detected new or changed devices on smithi171 2024-04-26T04:38:12.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:12.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:12.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:12.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:12.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: Adjusting osd_memory_target on smithi171 to 8544M 2024-04-26T04:38:12.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:12.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:12.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:12.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:12.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:12.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:12.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:12.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:12.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: pgmap v136: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:12.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:38:12.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:38:12.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:38:12.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T04:38:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: Detected new or changed devices on smithi171 2024-04-26T04:38:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: Adjusting osd_memory_target on smithi171 to 8544M 2024-04-26T04:38:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: pgmap v136: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:38:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:38:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:38:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:11 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:13.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:13 smithi156 ceph-mon[27570]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T04:38:13.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:13 smithi156 ceph-mon[27570]: Cluster is now healthy 2024-04-26T04:38:13.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:13 smithi156 ceph-mon[39130]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T04:38:13.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:13 smithi156 ceph-mon[39130]: Cluster is now healthy 2024-04-26T04:38:13.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:13 smithi171 ceph-mon[31139]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T04:38:13.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:13 smithi171 ceph-mon[31139]: Cluster is now healthy 2024-04-26T04:38:14.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:14 smithi156 ceph-mon[27570]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-04-26T04:38:14.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:14 smithi156 ceph-mon[27570]: from='client.? 172.21.15.171:0/3626746318' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1fd019a-372d-4b4e-bede-7659038df10a"}]: dispatch 2024-04-26T04:38:14.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:14 smithi156 ceph-mon[27570]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1fd019a-372d-4b4e-bede-7659038df10a"}]: dispatch 2024-04-26T04:38:14.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:14 smithi156 ceph-mon[27570]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f1fd019a-372d-4b4e-bede-7659038df10a"}]': finished 2024-04-26T04:38:14.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:14 smithi156 ceph-mon[27570]: osdmap e43: 7 total, 6 up, 7 in 2024-04-26T04:38:14.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:14 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:14.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:14 smithi156 ceph-mon[39130]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-04-26T04:38:14.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:14 smithi156 ceph-mon[39130]: from='client.? 172.21.15.171:0/3626746318' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1fd019a-372d-4b4e-bede-7659038df10a"}]: dispatch 2024-04-26T04:38:14.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:14 smithi156 ceph-mon[39130]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1fd019a-372d-4b4e-bede-7659038df10a"}]: dispatch 2024-04-26T04:38:14.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:14 smithi156 ceph-mon[39130]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f1fd019a-372d-4b4e-bede-7659038df10a"}]': finished 2024-04-26T04:38:14.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:14 smithi156 ceph-mon[39130]: osdmap e43: 7 total, 6 up, 7 in 2024-04-26T04:38:14.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:14 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:14.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:14 smithi171 ceph-mon[31139]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-04-26T04:38:14.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:14 smithi171 ceph-mon[31139]: from='client.? 172.21.15.171:0/3626746318' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1fd019a-372d-4b4e-bede-7659038df10a"}]: dispatch 2024-04-26T04:38:14.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:14 smithi171 ceph-mon[31139]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1fd019a-372d-4b4e-bede-7659038df10a"}]: dispatch 2024-04-26T04:38:14.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:14 smithi171 ceph-mon[31139]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f1fd019a-372d-4b4e-bede-7659038df10a"}]': finished 2024-04-26T04:38:14.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:14 smithi171 ceph-mon[31139]: osdmap e43: 7 total, 6 up, 7 in 2024-04-26T04:38:14.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:14 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:15.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:15 smithi156 ceph-mon[27570]: from='client.? 172.21.15.171:0/826483344' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:38:15.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:15 smithi156 ceph-mon[39130]: from='client.? 172.21.15.171:0/826483344' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:38:15.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:15 smithi171 ceph-mon[31139]: from='client.? 172.21.15.171:0/826483344' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:38:16.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:16 smithi156 ceph-mon[27570]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:38:16.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:16 smithi156 ceph-mon[39130]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:38:16.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:16 smithi171 ceph-mon[31139]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:38:18.087 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:18 smithi171 ceph-mon[31139]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-04-26T04:38:18.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:18 smithi156 ceph-mon[27570]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-04-26T04:38:18.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:18 smithi156 ceph-mon[39130]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-04-26T04:38:20.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:20 smithi171 ceph-mon[31139]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-04-26T04:38:20.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:20 smithi156 ceph-mon[27570]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-04-26T04:38:20.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:20 smithi156 ceph-mon[39130]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-04-26T04:38:21.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:38:21.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:21 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:21.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:21 smithi156 ceph-mon[27570]: Deploying daemon osd.6 on smithi171 2024-04-26T04:38:21.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:21 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:38:21.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:21 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:21.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:21 smithi156 ceph-mon[39130]: Deploying daemon osd.6 on smithi171 2024-04-26T04:38:21.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:38:21.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:21 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:21.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:21 smithi171 ceph-mon[31139]: Deploying daemon osd.6 on smithi171 2024-04-26T04:38:22.067 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:22 smithi171 ceph-mon[31139]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:38:22.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:22 smithi156 ceph-mon[27570]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:38:22.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:22 smithi156 ceph-mon[39130]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:38:24.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:24 smithi156 ceph-mon[27570]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:24.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:24 smithi156 ceph-mon[39130]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:24.344 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:24 smithi171 ceph-mon[31139]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:26.163 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:26 smithi171 ceph-mon[31139]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:26.163 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:26 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:26.163 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:26 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:26.164 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:26 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:26.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:26 smithi156 ceph-mon[27570]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:26.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:26 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:26.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:26 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:26.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:26 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:26.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:26 smithi156 ceph-mon[39130]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:26.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:26 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:26.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:26 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:26.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:26 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:28.215 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:28 smithi171 ceph-mon[31139]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:28.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:28 smithi156 ceph-mon[27570]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:28.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:28 smithi156 ceph-mon[39130]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:29.223 INFO:teuthology.orchestra.run.smithi171.stdout:Created osd(s) 6 on host 'smithi171' 2024-04-26T04:38:29.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:29 smithi171 ceph-mon[31139]: from='osd.6 [v2:172.21.15.171:6816/2328644410,v1:172.21.15.171:6817/2328644410]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:38:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:29 smithi171 ceph-mon[31139]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:38:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:29 smithi171 ceph-mon[31139]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:29 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:29.799 DEBUG:teuthology.orchestra.run.smithi171:osd.6> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.6.service 2024-04-26T04:38:29.801 INFO:tasks.cephadm:Deploying osd.7 on smithi171 with /dev/vg_nvme/lv_1... 2024-04-26T04:38:29.802 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-26T04:38:29.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:29.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:29.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:29.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:29.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:29.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[39130]: from='osd.6 [v2:172.21.15.171:6816/2328644410,v1:172.21.15.171:6817/2328644410]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:38:29.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[39130]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:38:29.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[39130]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:29.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:29.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:29.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:29.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:29.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:29.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:29.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:29.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:29.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[27570]: from='osd.6 [v2:172.21.15.171:6816/2328644410,v1:172.21.15.171:6817/2328644410]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:38:29.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[27570]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:38:29.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[27570]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:29.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:29.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:29.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:29 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:30.585 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:38:30.669 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:30 smithi171 ceph-mon[31139]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T04:38:30.669 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:30 smithi171 ceph-mon[31139]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T04:38:30.669 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:30 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:30.669 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:30 smithi171 ceph-mon[31139]: from='osd.6 [v2:172.21.15.171:6816/2328644410,v1:172.21.15.171:6817/2328644410]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:30.669 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:30 smithi171 ceph-mon[31139]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:30.670 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:38:30 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:38:30.454+0000 7f0f9334b700 -1 osd.6 0 waiting for initial osdmap 2024-04-26T04:38:30.670 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:38:30 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:38:30.460+0000 7f0f8d144700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:38:30.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:30 smithi156 ceph-mon[27570]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T04:38:30.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:30 smithi156 ceph-mon[27570]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T04:38:30.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:30 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:30.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:30 smithi156 ceph-mon[27570]: from='osd.6 [v2:172.21.15.171:6816/2328644410,v1:172.21.15.171:6817/2328644410]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:30.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:30 smithi156 ceph-mon[27570]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:30.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:30 smithi156 ceph-mon[39130]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T04:38:30.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:30 smithi156 ceph-mon[39130]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T04:38:30.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:30 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:30.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:30 smithi156 ceph-mon[39130]: from='osd.6 [v2:172.21.15.171:6816/2328644410,v1:172.21.15.171:6817/2328644410]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:30.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:30 smithi156 ceph-mon[39130]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:31.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:31 smithi171 ceph-mon[31139]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-04-26T04:38:31.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:31 smithi171 ceph-mon[31139]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T04:38:31.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:31 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:31.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:31 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:31.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:31 smithi171 ceph-mon[31139]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:31.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:31 smithi156 ceph-mon[27570]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-04-26T04:38:31.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:31 smithi156 ceph-mon[27570]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T04:38:31.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:31 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:31.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:31 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:31.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:31 smithi156 ceph-mon[27570]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:31.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:31 smithi156 ceph-mon[39130]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-04-26T04:38:31.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:31 smithi156 ceph-mon[39130]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T04:38:31.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:31 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:31.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:31 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:31.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:31 smithi156 ceph-mon[39130]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:38:32.732 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:32 smithi171 ceph-mon[31139]: purged_snaps scrub starts 2024-04-26T04:38:32.733 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:32 smithi171 ceph-mon[31139]: purged_snaps scrub ok 2024-04-26T04:38:32.733 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:32 smithi171 ceph-mon[31139]: osd.6 [v2:172.21.15.171:6816/2328644410,v1:172.21.15.171:6817/2328644410] boot 2024-04-26T04:38:32.733 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:32 smithi171 ceph-mon[31139]: osdmap e46: 7 total, 7 up, 7 in 2024-04-26T04:38:32.733 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:32 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:32.733 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:32 smithi171 ceph-mon[31139]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T04:38:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:32 smithi156 ceph-mon[27570]: purged_snaps scrub starts 2024-04-26T04:38:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:32 smithi156 ceph-mon[27570]: purged_snaps scrub ok 2024-04-26T04:38:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:32 smithi156 ceph-mon[27570]: osd.6 [v2:172.21.15.171:6816/2328644410,v1:172.21.15.171:6817/2328644410] boot 2024-04-26T04:38:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:32 smithi156 ceph-mon[27570]: osdmap e46: 7 total, 7 up, 7 in 2024-04-26T04:38:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:32 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:32 smithi156 ceph-mon[27570]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T04:38:32.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:32 smithi156 ceph-mon[39130]: purged_snaps scrub starts 2024-04-26T04:38:32.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:32 smithi156 ceph-mon[39130]: purged_snaps scrub ok 2024-04-26T04:38:32.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:32 smithi156 ceph-mon[39130]: osd.6 [v2:172.21.15.171:6816/2328644410,v1:172.21.15.171:6817/2328644410] boot 2024-04-26T04:38:32.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:32 smithi156 ceph-mon[39130]: osdmap e46: 7 total, 7 up, 7 in 2024-04-26T04:38:32.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:32 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:32.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:32 smithi156 ceph-mon[39130]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T04:38:33.688 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:33 smithi171 ceph-mon[31139]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T04:38:33.689 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:33 smithi171 ceph-mon[31139]: Detected new or changed devices on smithi171 2024-04-26T04:38:33.689 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:33.689 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:33.689 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:33.689 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:33.689 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:33.689 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:33 smithi171 ceph-mon[31139]: Adjusting osd_memory_target on smithi171 to 5696M 2024-04-26T04:38:33.689 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:33.689 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:33.689 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:33.689 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:33 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:33.689 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:33 smithi171 ceph-mon[31139]: pgmap v153: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:34.043 INFO:teuthology.orchestra.run.smithi171.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-26T04:38:34.044 INFO:teuthology.orchestra.run.smithi171.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-26T04:38:34.044 INFO:teuthology.orchestra.run.smithi171.stderr: stderr: 10+0 records in 2024-04-26T04:38:34.044 INFO:teuthology.orchestra.run.smithi171.stderr:10+0 records out 2024-04-26T04:38:34.044 INFO:teuthology.orchestra.run.smithi171.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0132601 s, 791 MB/s 2024-04-26T04:38:34.044 INFO:teuthology.orchestra.run.smithi171.stderr: stderr: 2024-04-26T04:38:34.044 INFO:teuthology.orchestra.run.smithi171.stderr:--> Zapping successful for: 2024-04-26T04:38:34.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[27570]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T04:38:34.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[27570]: Detected new or changed devices on smithi171 2024-04-26T04:38:34.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:34.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:34.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:34.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:34.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:34.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[27570]: Adjusting osd_memory_target on smithi171 to 5696M 2024-04-26T04:38:34.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:34.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:34.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:34.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:34.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[27570]: pgmap v153: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:34.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[39130]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T04:38:34.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[39130]: Detected new or changed devices on smithi171 2024-04-26T04:38:34.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:34.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:34.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:34.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:34.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:34.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[39130]: Adjusting osd_memory_target on smithi171 to 5696M 2024-04-26T04:38:34.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:34.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:34.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:34.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:34.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:33 smithi156 ceph-mon[39130]: pgmap v153: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:34.701 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi171:vg_nvme/lv_1 2024-04-26T04:38:34.953 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:38:34.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:34 smithi171 ceph-mon[31139]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T04:38:35.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:34 smithi156 ceph-mon[27570]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T04:38:35.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:34 smithi156 ceph-mon[39130]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T04:38:36.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:35 smithi156 ceph-mon[27570]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T04:38:36.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:35 smithi156 ceph-mon[27570]: pgmap v156: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:36.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:35 smithi156 ceph-mon[39130]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T04:38:36.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:35 smithi156 ceph-mon[39130]: pgmap v156: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:36.182 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:35 smithi171 ceph-mon[31139]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T04:38:36.182 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:35 smithi171 ceph-mon[31139]: pgmap v156: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:37.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:36 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:38:37.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:36 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:38:37.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:36 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:37.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:36 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:38:37.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:36 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:38:37.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:36 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:37.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:36 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:38:37.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:36 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:38:37.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:36 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:38.244 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:37 smithi156 ceph-mon[27570]: from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:38:38.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:37 smithi156 ceph-mon[27570]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-04-26T04:38:38.245 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:37 smithi156 ceph-mon[39130]: from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:38:38.245 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:37 smithi156 ceph-mon[39130]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-04-26T04:38:38.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:37 smithi171 ceph-mon[31139]: from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:38:38.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:37 smithi171 ceph-mon[31139]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-04-26T04:38:40.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:40 smithi156 ceph-mon[27570]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-26T04:38:40.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:40 smithi156 ceph-mon[27570]: from='client.? 172.21.15.171:0/1260386858' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bee1398e-7912-458a-9f8b-279b84a22566"}]: dispatch 2024-04-26T04:38:40.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:40 smithi156 ceph-mon[27570]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bee1398e-7912-458a-9f8b-279b84a22566"}]: dispatch 2024-04-26T04:38:40.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:40 smithi156 ceph-mon[27570]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bee1398e-7912-458a-9f8b-279b84a22566"}]': finished 2024-04-26T04:38:40.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:40 smithi156 ceph-mon[27570]: osdmap e51: 8 total, 7 up, 8 in 2024-04-26T04:38:40.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:40 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:40.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:40 smithi156 ceph-mon[39130]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-26T04:38:40.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:40 smithi156 ceph-mon[39130]: from='client.? 172.21.15.171:0/1260386858' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bee1398e-7912-458a-9f8b-279b84a22566"}]: dispatch 2024-04-26T04:38:40.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:40 smithi156 ceph-mon[39130]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bee1398e-7912-458a-9f8b-279b84a22566"}]: dispatch 2024-04-26T04:38:40.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:40 smithi156 ceph-mon[39130]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bee1398e-7912-458a-9f8b-279b84a22566"}]': finished 2024-04-26T04:38:40.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:40 smithi156 ceph-mon[39130]: osdmap e51: 8 total, 7 up, 8 in 2024-04-26T04:38:40.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:40 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:40.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:40 smithi171 ceph-mon[31139]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-26T04:38:40.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:40 smithi171 ceph-mon[31139]: from='client.? 172.21.15.171:0/1260386858' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bee1398e-7912-458a-9f8b-279b84a22566"}]: dispatch 2024-04-26T04:38:40.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:40 smithi171 ceph-mon[31139]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bee1398e-7912-458a-9f8b-279b84a22566"}]: dispatch 2024-04-26T04:38:40.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:40 smithi171 ceph-mon[31139]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bee1398e-7912-458a-9f8b-279b84a22566"}]': finished 2024-04-26T04:38:40.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:40 smithi171 ceph-mon[31139]: osdmap e51: 8 total, 7 up, 8 in 2024-04-26T04:38:40.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:40 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:41.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:41 smithi156 ceph-mon[27570]: from='client.? 172.21.15.171:0/29106329' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:38:41.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:41 smithi156 ceph-mon[39130]: from='client.? 172.21.15.171:0/29106329' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:38:41.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:41 smithi171 ceph-mon[31139]: from='client.? 172.21.15.171:0/29106329' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:38:42.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:42 smithi156 ceph-mon[27570]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-04-26T04:38:42.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:42 smithi156 ceph-mon[39130]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-04-26T04:38:42.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:42 smithi171 ceph-mon[31139]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-04-26T04:38:44.297 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:44 smithi171 ceph-mon[31139]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-26T04:38:44.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:44 smithi156 ceph-mon[27570]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-26T04:38:44.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:44 smithi156 ceph-mon[39130]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-26T04:38:46.299 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:46 smithi171 ceph-mon[31139]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:38:46.299 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:46 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:38:46.299 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:46 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:46.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:46 smithi156 ceph-mon[27570]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:38:46.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:46 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:38:46.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:46 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:46.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:46 smithi156 ceph-mon[39130]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:38:46.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:46 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:38:46.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:46 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:47.175 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:47 smithi171 ceph-mon[31139]: Deploying daemon osd.7 on smithi171 2024-04-26T04:38:47.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:47 smithi156 ceph-mon[27570]: Deploying daemon osd.7 on smithi171 2024-04-26T04:38:47.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:47 smithi156 ceph-mon[39130]: Deploying daemon osd.7 on smithi171 2024-04-26T04:38:48.215 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:48 smithi171 ceph-mon[31139]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:48.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:48 smithi156 ceph-mon[27570]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:48.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:48 smithi156 ceph-mon[39130]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:50.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:50 smithi156 ceph-mon[39130]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:50.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:50 smithi156 ceph-mon[27570]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:50.356 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:50 smithi171 ceph-mon[31139]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:52.307 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:52 smithi171 ceph-mon[31139]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:52.307 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:52 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:52.307 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:52 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:52.307 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:52 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:52.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:52 smithi156 ceph-mon[27570]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:52.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:52 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:52.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:52 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:52.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:52 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:52.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:52 smithi156 ceph-mon[39130]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:52.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:52 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:52.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:52 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:52.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:52 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:54.209 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:54 smithi171 ceph-mon[31139]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:54.210 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:54 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:54.210 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:54 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:54.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:54 smithi156 ceph-mon[27570]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:54.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:54 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:54.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:54 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:54.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:54 smithi156 ceph-mon[39130]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:54.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:54 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:54.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:54 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:54.600 INFO:teuthology.orchestra.run.smithi171.stdout:Created osd(s) 7 on host 'smithi171' 2024-04-26T04:38:55.176 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:55 smithi171 ceph-mon[31139]: from='osd.7 [v2:172.21.15.171:6824/2507226151,v1:172.21.15.171:6825/2507226151]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:38:55.176 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:55 smithi171 ceph-mon[31139]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:38:55.176 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:55 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:55.176 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:55 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:55.176 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:55 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:55.176 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:55 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:55.176 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:55 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:55.176 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:55 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:55.199 DEBUG:teuthology.orchestra.run.smithi171:osd.7> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.7.service 2024-04-26T04:38:55.201 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-26T04:38:55.201 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-26T04:38:55.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:55 smithi156 ceph-mon[27570]: from='osd.7 [v2:172.21.15.171:6824/2507226151,v1:172.21.15.171:6825/2507226151]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:38:55.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:55 smithi156 ceph-mon[27570]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:38:55.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:55 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:55.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:55 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:55.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:55 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:55.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:55 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:55.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:55 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:55.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:55 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:55.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:55 smithi156 ceph-mon[39130]: from='osd.7 [v2:172.21.15.171:6824/2507226151,v1:172.21.15.171:6825/2507226151]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:38:55.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:55 smithi156 ceph-mon[39130]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:38:55.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:55 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:55.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:55 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:55.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:55 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:55.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:55 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:55.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:55 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:55.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:55 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:55.461 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:38:56.189 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:56 smithi156 ceph-mon[27570]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:56.190 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:56 smithi156 ceph-mon[27570]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T04:38:56.190 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:56 smithi156 ceph-mon[27570]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T04:38:56.190 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:56 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:56.190 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:56 smithi156 ceph-mon[27570]: from='osd.7 [v2:172.21.15.171:6824/2507226151,v1:172.21.15.171:6825/2507226151]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:56.190 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:56 smithi156 ceph-mon[27570]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:56.190 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:56 smithi156 ceph-mon[39130]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:56.190 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:56 smithi156 ceph-mon[39130]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T04:38:56.190 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:56 smithi156 ceph-mon[39130]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T04:38:56.190 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:56 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:56.191 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:56 smithi156 ceph-mon[39130]: from='osd.7 [v2:172.21.15.171:6824/2507226151,v1:172.21.15.171:6825/2507226151]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:56.191 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:56 smithi156 ceph-mon[39130]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:56.432 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:56 smithi171 ceph-mon[31139]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:56.432 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:56 smithi171 ceph-mon[31139]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T04:38:56.432 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:56 smithi171 ceph-mon[31139]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T04:38:56.434 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:56 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:56.434 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:56 smithi171 ceph-mon[31139]: from='osd.7 [v2:172.21.15.171:6824/2507226151,v1:172.21.15.171:6825/2507226151]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:56.434 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:56 smithi171 ceph-mon[31139]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:38:56.435 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:38:56 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:38:56.021+0000 7fd05b3c0700 -1 osd.7 0 waiting for initial osdmap 2024-04-26T04:38:56.435 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:38:56 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:38:56.028+0000 7fd0579be700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:38:57.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:57 smithi156 ceph-mon[27570]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-04-26T04:38:57.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:57 smithi156 ceph-mon[27570]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T04:38:57.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:57 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:57.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:57 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:57.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:57 smithi156 ceph-mon[27570]: osd.7 [v2:172.21.15.171:6824/2507226151,v1:172.21.15.171:6825/2507226151] boot 2024-04-26T04:38:57.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:57 smithi156 ceph-mon[27570]: osdmap e54: 8 total, 8 up, 8 in 2024-04-26T04:38:57.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:57 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:57.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:57 smithi156 ceph-mon[39130]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-04-26T04:38:57.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:57 smithi156 ceph-mon[39130]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T04:38:57.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:57 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:57.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:57 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:57.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:57 smithi156 ceph-mon[39130]: osd.7 [v2:172.21.15.171:6824/2507226151,v1:172.21.15.171:6825/2507226151] boot 2024-04-26T04:38:57.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:57 smithi156 ceph-mon[39130]: osdmap e54: 8 total, 8 up, 8 in 2024-04-26T04:38:57.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:57 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:57.265 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:38:57.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:57 smithi171 ceph-mon[31139]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-04-26T04:38:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:57 smithi171 ceph-mon[31139]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T04:38:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:57 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:57 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:57 smithi171 ceph-mon[31139]: osd.7 [v2:172.21.15.171:6824/2507226151,v1:172.21.15.171:6825/2507226151] boot 2024-04-26T04:38:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:57 smithi171 ceph-mon[31139]: osdmap e54: 8 total, 8 up, 8 in 2024-04-26T04:38:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:57 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:57.873 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":54,"num_osds":8,"num_up_osds":8,"osd_up_since":1714106336,"num_in_osds":8,"osd_in_since":1714106319,"num_remapped_pgs":1} 2024-04-26T04:38:57.873 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:38:58.125 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:38:58.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:58 smithi171 ceph-mon[31139]: purged_snaps scrub starts 2024-04-26T04:38:58.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:58 smithi171 ceph-mon[31139]: purged_snaps scrub ok 2024-04-26T04:38:58.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:58 smithi171 ceph-mon[31139]: pgmap v171: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:58.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:58 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1619942430' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:38:58.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:58 smithi171 ceph-mon[31139]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T04:38:58.247 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:58 smithi156 ceph-mon[27570]: purged_snaps scrub starts 2024-04-26T04:38:58.247 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:58 smithi156 ceph-mon[27570]: purged_snaps scrub ok 2024-04-26T04:38:58.248 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:58 smithi156 ceph-mon[27570]: pgmap v171: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:58.248 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:58 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1619942430' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:38:58.248 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:58 smithi156 ceph-mon[27570]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T04:38:58.248 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:58 smithi156 ceph-mon[39130]: purged_snaps scrub starts 2024-04-26T04:38:58.248 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:58 smithi156 ceph-mon[39130]: purged_snaps scrub ok 2024-04-26T04:38:58.248 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:58 smithi156 ceph-mon[39130]: pgmap v171: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:58.248 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:58 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1619942430' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:38:58.248 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:58 smithi156 ceph-mon[39130]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T04:38:59.307 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[27570]: Detected new or changed devices on smithi171 2024-04-26T04:38:59.307 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:59.307 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:59.307 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:59.307 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:59.307 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:59.307 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:59.307 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[27570]: Adjusting osd_memory_target on smithi171 to 4272M 2024-04-26T04:38:59.307 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:59.307 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:59.307 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:59.307 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:59.308 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[27570]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T04:38:59.308 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[39130]: Detected new or changed devices on smithi171 2024-04-26T04:38:59.308 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:59.308 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:59.308 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:59.308 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:59.308 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:59.308 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:59.308 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[39130]: Adjusting osd_memory_target on smithi171 to 4272M 2024-04-26T04:38:59.308 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:59.308 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:59.308 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:59.309 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:59.309 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:38:59 smithi156 ceph-mon[39130]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T04:38:59.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:59 smithi171 ceph-mon[31139]: Detected new or changed devices on smithi171 2024-04-26T04:38:59.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:59.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:59.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:59.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:59.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:59.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:59.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:59 smithi171 ceph-mon[31139]: Adjusting osd_memory_target on smithi171 to 4272M 2024-04-26T04:38:59.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:59.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:59.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:59.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:59 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:38:59.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:38:59 smithi171 ceph-mon[31139]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T04:39:00.040 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:39:00.040 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":57,"fsid":"110027f2-0386-11ef-bc93-c7b262605968","created":"2024-04-26T04:33:46.638460+0000","modified":"2024-04-26T04:38:59.616576+0000","last_up_change":"2024-04-26T04:38:56.608997+0000","last_in_change":"2024-04-26T04:38:39.401044+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:36:55.213219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"6b59fe49-0273-4115-91d9-6faea7d37616","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6803","nonce":3436276177}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6805","nonce":3436276177}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6809","nonce":3436276177}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6807","nonce":3436276177}]},"public_addr":"172.21.15.156:6803/3436276177","cluster_addr":"172.21.15.156:6805/3436276177","heartbeat_back_addr":"172.21.15.156:6809/3436276177","heartbeat_front_addr":"172.21.15.156:6807/3436276177","state":["exists","up"]},{"osd":1,"uuid":"c9673007-264f-4d7c-8234-bfd514e70d2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6811","nonce":273749030}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6813","nonce":273749030}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6817","nonce":273749030}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6815","nonce":273749030}]},"public_addr":"172.21.15.156:6811/273749030","cluster_addr":"172.21.15.156:6813/273749030","heartbeat_back_addr":"172.21.15.156:6817/273749030","heartbeat_front_addr":"172.21.15.156:6815/273749030","state":["exists","up"]},{"osd":2,"uuid":"d0051bbe-c0cd-42f4-9d8c-6b0a3e71886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6819","nonce":3653838089}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6821","nonce":3653838089}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6825","nonce":3653838089}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6823","nonce":3653838089}]},"public_addr":"172.21.15.156:6819/3653838089","cluster_addr":"172.21.15.156:6821/3653838089","heartbeat_back_addr":"172.21.15.156:6825/3653838089","heartbeat_front_addr":"172.21.15.156:6823/3653838089","state":["exists","up"]},{"osd":3,"uuid":"479e109c-1e4a-4124-9442-f85d63df6d04","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6827","nonce":2175305841}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6829","nonce":2175305841}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6832","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6833","nonce":2175305841}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6831","nonce":2175305841}]},"public_addr":"172.21.15.156:6827/2175305841","cluster_addr":"172.21.15.156:6829/2175305841","heartbeat_back_addr":"172.21.15.156:6833/2175305841","heartbeat_front_addr":"172.21.15.156:6831/2175305841","state":["exists","up"]},{"osd":4,"uuid":"b090dbe4-ed23-4015-a864-074aa64cace4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6801","nonce":2255415442}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6803","nonce":2255415442}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6807","nonce":2255415442}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6805","nonce":2255415442}]},"public_addr":"172.21.15.171:6801/2255415442","cluster_addr":"172.21.15.171:6803/2255415442","heartbeat_back_addr":"172.21.15.171:6807/2255415442","heartbeat_front_addr":"172.21.15.171:6805/2255415442","state":["exists","up"]},{"osd":5,"uuid":"f506481f-428b-40d6-a54b-8ef411672b53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6809","nonce":2373798416}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6811","nonce":2373798416}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6815","nonce":2373798416}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6813","nonce":2373798416}]},"public_addr":"172.21.15.171:6809/2373798416","cluster_addr":"172.21.15.171:6811/2373798416","heartbeat_back_addr":"172.21.15.171:6815/2373798416","heartbeat_front_addr":"172.21.15.171:6813/2373798416","state":["exists","up"]},{"osd":6,"uuid":"f1fd019a-372d-4b4e-bede-7659038df10a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6817","nonce":2328644410}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6819","nonce":2328644410}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6823","nonce":2328644410}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6821","nonce":2328644410}]},"public_addr":"172.21.15.171:6817/2328644410","cluster_addr":"172.21.15.171:6819/2328644410","heartbeat_back_addr":"172.21.15.171:6823/2328644410","heartbeat_front_addr":"172.21.15.171:6821/2328644410","state":["exists","up"]},{"osd":7,"uuid":"bee1398e-7912-458a-9f8b-279b84a22566","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6825","nonce":2507226151}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6827","nonce":2507226151}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6831","nonce":2507226151}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6829","nonce":2507226151}]},"public_addr":"172.21.15.171:6825/2507226151","cluster_addr":"172.21.15.171:6827/2507226151","heartbeat_back_addr":"172.21.15.171:6831/2507226151","heartbeat_front_addr":"172.21.15.171:6829/2507226151","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:03.400774+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:28.117542+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:52.563066+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:17.387470+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:40.460850+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:05.110287+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:29.602919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:55.143996+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:6801/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:6800/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/67599134":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/4143435073":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6801/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/402721227":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/2398977550":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6800/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/1144243691":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/4047047180":"2024-04-27T04:34:42.908359+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:39:00.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:00 smithi171 ceph-mon[31139]: pgmap v174: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:00.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:00 smithi171 ceph-mon[31139]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T04:39:00.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:00 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2505236323' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:00.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:00 smithi156 ceph-mon[27570]: pgmap v174: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:00.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:00 smithi156 ceph-mon[27570]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T04:39:00.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:00 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2505236323' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:00.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:00 smithi156 ceph-mon[39130]: pgmap v174: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:00.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:00 smithi156 ceph-mon[39130]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T04:39:00.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:00 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2505236323' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:00.705 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-26T04:36:55.213219+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-26T04:39:00.706 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-26T04:39:00.955 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:01.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:01 smithi171 ceph-mon[31139]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T04:39:02.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:01 smithi171 ceph-mon[31139]: pgmap v177: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:02.072 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:01 smithi156 ceph-mon[27570]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T04:39:02.072 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:01 smithi156 ceph-mon[27570]: pgmap v177: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:02.072 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:01 smithi156 ceph-mon[39130]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T04:39:02.073 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:01 smithi156 ceph-mon[39130]: pgmap v177: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:02.766 INFO:teuthology.orchestra.run.smithi156.stdout:pg_num: 1 2024-04-26T04:39:03.063 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:02 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3060136735' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T04:39:03.063 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:02 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3060136735' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T04:39:03.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:02 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3060136735' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T04:39:03.348 INFO:tasks.cephadm:Adding prometheus.a on smithi171 2024-04-26T04:39:03.348 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph orch apply prometheus '1;smithi171=a' 2024-04-26T04:39:03.612 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:39:04.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:03 smithi171 ceph-mon[31139]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-04-26T04:39:04.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:03 smithi156 ceph-mon[27570]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-04-26T04:39:04.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:03 smithi156 ceph-mon[39130]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-04-26T04:39:05.353 INFO:teuthology.orchestra.run.smithi171.stdout:Scheduled prometheus update... 2024-04-26T04:39:06.028 DEBUG:teuthology.orchestra.run.smithi171:prometheus.a> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@prometheus.a.service 2024-04-26T04:39:06.031 INFO:tasks.cephadm:Adding node-exporter.a on smithi156 2024-04-26T04:39:06.031 INFO:tasks.cephadm:Adding node-exporter.b on smithi171 2024-04-26T04:39:06.031 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph orch apply node-exporter '2;smithi156=a;smithi171=b' 2024-04-26T04:39:06.344 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:39:06.612 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:06 smithi171 ceph-mon[31139]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-26T04:39:06.612 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:06 smithi171 ceph-mon[31139]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi171=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:06.612 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:06 smithi171 ceph-mon[31139]: Saving service prometheus spec with placement smithi171=a;count:1 2024-04-26T04:39:06.612 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:06 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:06.612 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:06 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:06.612 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:06 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:06.612 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:06 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:39:06.612 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:06 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:06.612 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:06 smithi171 ceph-mon[31139]: Deploying daemon prometheus.a on smithi171 2024-04-26T04:39:06.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[27570]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-26T04:39:06.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[27570]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi171=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:06.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[27570]: Saving service prometheus spec with placement smithi171=a;count:1 2024-04-26T04:39:06.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:06.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:06.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:06.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:39:06.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:06.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[27570]: Deploying daemon prometheus.a on smithi171 2024-04-26T04:39:06.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[39130]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-26T04:39:06.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[39130]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi171=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:06.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[39130]: Saving service prometheus spec with placement smithi171=a;count:1 2024-04-26T04:39:06.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:06.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:06.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:06.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:39:06.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:06.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:06 smithi156 ceph-mon[39130]: Deploying daemon prometheus.a on smithi171 2024-04-26T04:39:08.585 INFO:teuthology.orchestra.run.smithi171.stdout:Scheduled node-exporter update... 2024-04-26T04:39:08.646 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:08 smithi156 ceph-mon[27570]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:39:08.646 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:08 smithi156 ceph-mon[39130]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:39:08.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:08 smithi171 ceph-mon[31139]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:39:09.764 DEBUG:teuthology.orchestra.run.smithi156:node-exporter.a> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@node-exporter.a.service 2024-04-26T04:39:09.767 DEBUG:teuthology.orchestra.run.smithi171:node-exporter.b> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@node-exporter.b.service 2024-04-26T04:39:09.770 INFO:tasks.cephadm:Adding alertmanager.a on smithi156 2024-04-26T04:39:09.770 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph orch apply alertmanager '1;smithi156=a' 2024-04-26T04:39:09.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:09 smithi156 ceph-mon[27570]: from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi156=a;smithi171=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:09.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:09 smithi156 ceph-mon[27570]: Saving service node-exporter spec with placement smithi156=a;smithi171=b;count:2 2024-04-26T04:39:09.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:09 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:09.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:09 smithi156 ceph-mon[27570]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-26T04:39:09.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:09 smithi156 ceph-mon[39130]: from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi156=a;smithi171=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:09.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:09 smithi156 ceph-mon[39130]: Saving service node-exporter spec with placement smithi156=a;smithi171=b;count:2 2024-04-26T04:39:09.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:09 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:09.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:09 smithi156 ceph-mon[39130]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-26T04:39:09.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:09 smithi171 ceph-mon[31139]: from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi156=a;smithi171=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:09.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:09 smithi171 ceph-mon[31139]: Saving service node-exporter spec with placement smithi156=a;smithi171=b;count:2 2024-04-26T04:39:10.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:09 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:10.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:09 smithi171 ceph-mon[31139]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-26T04:39:10.080 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:39:12.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:12 smithi156 ceph-mon[27570]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-04-26T04:39:12.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:12 smithi156 ceph-mon[39130]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-04-26T04:39:12.352 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:12 smithi171 ceph-mon[31139]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-04-26T04:39:12.586 INFO:teuthology.orchestra.run.smithi171.stdout:Scheduled alertmanager update... 2024-04-26T04:39:13.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:13 smithi156 ceph-mon[39130]: from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi156=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:13.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:13 smithi156 ceph-mon[39130]: Saving service alertmanager spec with placement smithi156=a;count:1 2024-04-26T04:39:13.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:13 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:13.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:13 smithi156 ceph-mon[39130]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-26T04:39:13.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:13 smithi156 ceph-mon[27570]: from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi156=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:13.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:13 smithi156 ceph-mon[27570]: Saving service alertmanager spec with placement smithi156=a;count:1 2024-04-26T04:39:13.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:13 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:13.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:13 smithi156 ceph-mon[27570]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-26T04:39:13.874 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:13 smithi171 ceph-mon[31139]: from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi156=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:13.875 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:13 smithi171 ceph-mon[31139]: Saving service alertmanager spec with placement smithi156=a;count:1 2024-04-26T04:39:13.875 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:13 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:13.875 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:13 smithi171 ceph-mon[31139]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-26T04:39:14.411 DEBUG:teuthology.orchestra.run.smithi156:alertmanager.a> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@alertmanager.a.service 2024-04-26T04:39:14.414 INFO:tasks.cephadm:Adding grafana.a on smithi171 2024-04-26T04:39:14.414 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph orch apply grafana '1;smithi171=a' 2024-04-26T04:39:14.819 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:39:16.166 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:16 smithi171 ceph-mon[31139]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:16.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:16 smithi156 ceph-mon[27570]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:16.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:16 smithi156 ceph-mon[39130]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:18.071 INFO:teuthology.orchestra.run.smithi171.stdout:Scheduled grafana update... 2024-04-26T04:39:18.281 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:18 smithi171 ceph-mon[31139]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:18.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:18 smithi156 ceph-mon[27570]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:18.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:18 smithi156 ceph-mon[39130]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:19.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:19 smithi156 ceph-mon[27570]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi171=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:19.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:19 smithi156 ceph-mon[27570]: Saving service grafana spec with placement smithi171=a;count:1 2024-04-26T04:39:19.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:19 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:19.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:19 smithi156 ceph-mon[39130]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi171=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:19.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:19 smithi156 ceph-mon[39130]: Saving service grafana spec with placement smithi171=a;count:1 2024-04-26T04:39:19.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:19 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:19.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:19 smithi171 ceph-mon[31139]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi171=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:19.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:19 smithi171 ceph-mon[31139]: Saving service grafana spec with placement smithi171=a;count:1 2024-04-26T04:39:19.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:19 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:19.935 DEBUG:teuthology.orchestra.run.smithi171:grafana.a> sudo journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service 2024-04-26T04:39:19.938 INFO:tasks.cephadm:Setting up client nodes... 2024-04-26T04:39:19.938 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-26T04:39:20.196 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:20.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:20 smithi171 ceph-mon[31139]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:20.318 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:20 smithi156 ceph-mon[27570]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:20.318 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:20 smithi156 ceph-mon[39130]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:21.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:20 smithi171 systemd[1]: Starting Ceph prometheus.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:39:21.332 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 podman[60126]: 2024-04-26 04:39:20.966067693 +0000 UTC m=+0.018615458 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T04:39:21.332 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 podman[60126]: 2024-04-26 04:39:21.070085674 +0000 UTC m=+0.122633426 container create 276c3aacfbc140f4427776fd3863a10db03f85eb4e3ae92650b07a5bb0741e8e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:21.332 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 podman[60126]: 2024-04-26 04:39:21.332257909 +0000 UTC m=+0.384805661 container init 276c3aacfbc140f4427776fd3863a10db03f85eb4e3ae92650b07a5bb0741e8e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:21.750 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 podman[60126]: 2024-04-26 04:39:21.337398672 +0000 UTC m=+0.389946425 container start 276c3aacfbc140f4427776fd3863a10db03f85eb4e3ae92650b07a5bb0741e8e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:21.750 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.360Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T04:39:21.750 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.360Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T04:39:21.750 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.360Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi171 (none))" 2024-04-26T04:39:21.750 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.360Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T04:39:21.750 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.360Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T04:39:21.750 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.361Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T04:39:21.750 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.362Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T04:39:21.751 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.363Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T04:39:21.751 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.363Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T04:39:21.751 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.365Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T04:39:21.751 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.365Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.718µs 2024-04-26T04:39:21.751 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.365Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T04:39:21.751 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.365Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-04-26T04:39:21.751 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.365Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.704µs wal_replay_duration=163.395µs wbl_replay_duration=120ns total_replay_duration=203.198µs 2024-04-26T04:39:21.751 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.366Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T04:39:21.751 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.366Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T04:39:21.751 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.366Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T04:39:21.751 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.380Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.536663ms db_storage=784ns remote_storage=1.054µs web_handler=359ns query_engine=544ns scrape=444.81µs scrape_sd=15.095µs notify=751ns notify_sd=1.695µs rules=13.758088ms tracing=5.436µs 2024-04-26T04:39:21.751 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.380Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T04:39:21.751 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:39:21.380Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T04:39:21.752 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 bash[60126]: 276c3aacfbc140f4427776fd3863a10db03f85eb4e3ae92650b07a5bb0741e8e 2024-04-26T04:39:21.752 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:39:21 smithi171 systemd[1]: Started Ceph prometheus.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:39:22.319 INFO:teuthology.orchestra.run.smithi156.stdout:[client.0] 2024-04-26T04:39:22.319 INFO:teuthology.orchestra.run.smithi156.stdout: key = AQD6LytmYbHFEhAA1QazAD4xFfXBPypr6hyXuQ== 2024-04-26T04:39:22.331 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:22 smithi156 ceph-mon[39130]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:22.331 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:22 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:22.331 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:22 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:22.332 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:22 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:22.332 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:22 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T04:39:22.332 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:22 smithi156 ceph-mon[27570]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:22.332 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:22 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:22.332 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:22 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:22.332 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:22 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:22.332 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:22 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T04:39:22.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:22 smithi171 ceph-mon[31139]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:22.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:22 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:22.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:22 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:22.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:22 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' 2024-04-26T04:39:22.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:22 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T04:39:22.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:22 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ignoring --setuser ceph since I am not root 2024-04-26T04:39:22.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:22 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ignoring --setgroup ceph since I am not root 2024-04-26T04:39:22.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:22 smithi156 ceph-mgr[27844]: -- 172.21.15.156:0/405171071 <== mon.0 v2:172.21.15.156:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56137d7d09c0 con 0x56137d7a7000 2024-04-26T04:39:22.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:22 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:22.700+0000 7fac1fb5b1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:39:22.860 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:22 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ignoring --setuser ceph since I am not root 2024-04-26T04:39:22.861 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:22 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ignoring --setgroup ceph since I am not root 2024-04-26T04:39:22.861 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:22 smithi171 ceph-mgr[33263]: -- 172.21.15.171:0/583341691 <== mon.1 v2:172.21.15.171:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x5579fcd7ac60 con 0x5579fcd3c800 2024-04-26T04:39:22.861 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:22 smithi171 ceph-mgr[33263]: -- 172.21.15.171:0/583341691 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5579fcd669c0 con 0x5579fcd3c800 2024-04-26T04:39:22.861 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:22 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:22.702+0000 7f05f3c801c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:39:22.861 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:22 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:22.859+0000 7f05f3c801c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:39:23.080 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-04-26T04:39:23.080 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-26T04:39:23.080 DEBUG:teuthology.orchestra.run.smithi156:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-26T04:39:23.113 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:22 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:22.861+0000 7fac1fb5b1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:39:23.114 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:22 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:22.963+0000 7fac1fb5b1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:39:23.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:23 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3371269667' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:39:23.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:23 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3371269667' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:39:23.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:23 smithi156 ceph-mon[27570]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T04:39:23.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:23 smithi156 ceph-mon[27570]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T04:39:23.114 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:23 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3371269667' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:39:23.114 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:23 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3371269667' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:39:23.115 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:23 smithi156 ceph-mon[39130]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T04:39:23.115 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:23 smithi156 ceph-mon[39130]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T04:39:23.117 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-26T04:39:23.135 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:22 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:22.957+0000 7f05f3c801c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:39:23.135 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:23 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3371269667' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:39:23.135 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:23 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3371269667' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:39:23.135 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:23 smithi171 ceph-mon[31139]: from='mgr.14150 172.21.15.156:0/1637587179' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T04:39:23.135 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:23 smithi171 ceph-mon[31139]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T04:39:23.390 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:39:23.418 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:23 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:23.271+0000 7f05f3c801c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:39:23.418 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:23 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:23.364+0000 7f05f3c801c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:39:23.593 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:23.277+0000 7fac1fb5b1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:39:23.593 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:23 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:23.359+0000 7fac1fb5b1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:39:24.631 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:24 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:24.572+0000 7f05f3c801c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:39:24.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:24.505+0000 7fac1fb5b1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:39:24.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:24.671+0000 7fac1fb5b1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:39:24.907 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:24 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:24.759+0000 7f05f3c801c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:39:25.094 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:24.840+0000 7fac1fb5b1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:39:25.206 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:24 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:24.933+0000 7f05f3c801c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:39:25.206 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:25.204+0000 7f05f3c801c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:39:25.458 INFO:teuthology.orchestra.run.smithi171.stdout:[client.1] 2024-04-26T04:39:25.458 INFO:teuthology.orchestra.run.smithi171.stdout: key = AQD9LytmJnoYGxAA+csBFAjwgtyM2UoPmasSgQ== 2024-04-26T04:39:25.459 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:25.295+0000 7f05f3c801c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:39:25.459 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:25.388+0000 7f05f3c801c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:39:25.506 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:25 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:25.104+0000 7fac1fb5b1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:39:25.506 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:25 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:25.194+0000 7fac1fb5b1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:39:25.507 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:25 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:25.287+0000 7fac1fb5b1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:39:25.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:25.671+0000 7f05f3c801c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:39:25.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:25 smithi171 ceph-mon[31139]: from='client.? 172.21.15.171:0/1781927125' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:39:25.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:25 smithi171 ceph-mon[31139]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:39:25.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:25 smithi171 ceph-mon[31139]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:39:25.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:25 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:25.569+0000 7fac1fb5b1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:39:25.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:25 smithi156 ceph-mon[39130]: from='client.? 172.21.15.171:0/1781927125' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:39:25.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:25 smithi156 ceph-mon[39130]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:39:25.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:25 smithi156 ceph-mon[39130]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:39:25.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:25 smithi156 ceph-mon[27570]: from='client.? 172.21.15.171:0/1781927125' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:39:25.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:25 smithi156 ceph-mon[27570]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:39:25.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:25 smithi156 ceph-mon[27570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:39:26.122 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-04-26T04:39:26.122 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-26T04:39:26.122 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-26T04:39:26.198 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-04-26T04:39:26.198 DEBUG:teuthology.orchestra.run.smithi156:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-04-26T04:39:26.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:26 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:26.028+0000 7fac1fb5b1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:39:26.476 INFO:teuthology.misc.health.smithi156.stdout: 2024-04-26T04:39:26.476 INFO:teuthology.misc.health.smithi156.stdout:{"epoch":58,"fsid":"110027f2-0386-11ef-bc93-c7b262605968","created":"2024-04-26T04:33:46.638460+0000","modified":"2024-04-26T04:39:00.620612+0000","last_up_change":"2024-04-26T04:38:56.608997+0000","last_in_change":"2024-04-26T04:38:39.401044+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:36:55.213219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"6b59fe49-0273-4115-91d9-6faea7d37616","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6803","nonce":3436276177}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6805","nonce":3436276177}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6809","nonce":3436276177}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6807","nonce":3436276177}]},"public_addr":"172.21.15.156:6803/3436276177","cluster_addr":"172.21.15.156:6805/3436276177","heartbeat_back_addr":"172.21.15.156:6809/3436276177","heartbeat_front_addr":"172.21.15.156:6807/3436276177","state":["exists","up"]},{"osd":1,"uuid":"c9673007-264f-4d7c-8234-bfd514e70d2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6811","nonce":273749030}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6813","nonce":273749030}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6817","nonce":273749030}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6815","nonce":273749030}]},"public_addr":"172.21.15.156:6811/273749030","cluster_addr":"172.21.15.156:6813/273749030","heartbeat_back_addr":"172.21.15.156:6817/273749030","heartbeat_front_addr":"172.21.15.156:6815/273749030","state":["exists","up"]},{"osd":2,"uuid":"d0051bbe-c0cd-42f4-9d8c-6b0a3e71886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6819","nonce":3653838089}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6821","nonce":3653838089}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6825","nonce":3653838089}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6823","nonce":3653838089}]},"public_addr":"172.21.15.156:6819/3653838089","cluster_addr":"172.21.15.156:6821/3653838089","heartbeat_back_addr":"172.21.15.156:6825/3653838089","heartbeat_front_addr":"172.21.15.156:6823/3653838089","state":["exists","up"]},{"osd":3,"uuid":"479e109c-1e4a-4124-9442-f85d63df6d04","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6827","nonce":2175305841}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6829","nonce":2175305841}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6832","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6833","nonce":2175305841}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6831","nonce":2175305841}]},"public_addr":"172.21.15.156:6827/2175305841","cluster_addr":"172.21.15.156:6829/2175305841","heartbeat_back_addr":"172.21.15.156:6833/2175305841","heartbeat_front_addr":"172.21.15.156:6831/2175305841","state":["exists","up"]},{"osd":4,"uuid":"b090dbe4-ed23-4015-a864-074aa64cace4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6801","nonce":2255415442}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6803","nonce":2255415442}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6807","nonce":2255415442}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6805","nonce":2255415442}]},"public_addr":"172.21.15.171:6801/2255415442","cluster_addr":"172.21.15.171:6803/2255415442","heartbeat_back_addr":"172.21.15.171:6807/2255415442","heartbeat_front_addr":"172.21.15.171:6805/2255415442","state":["exists","up"]},{"osd":5,"uuid":"f506481f-428b-40d6-a54b-8ef411672b53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6809","nonce":2373798416}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6811","nonce":2373798416}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6815","nonce":2373798416}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6813","nonce":2373798416}]},"public_addr":"172.21.15.171:6809/2373798416","cluster_addr":"172.21.15.171:6811/2373798416","heartbeat_back_addr":"172.21.15.171:6815/2373798416","heartbeat_front_addr":"172.21.15.171:6813/2373798416","state":["exists","up"]},{"osd":6,"uuid":"f1fd019a-372d-4b4e-bede-7659038df10a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6817","nonce":2328644410}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6819","nonce":2328644410}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6823","nonce":2328644410}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6821","nonce":2328644410}]},"public_addr":"172.21.15.171:6817/2328644410","cluster_addr":"172.21.15.171:6819/2328644410","heartbeat_back_addr":"172.21.15.171:6823/2328644410","heartbeat_front_addr":"172.21.15.171:6821/2328644410","state":["exists","up"]},{"osd":7,"uuid":"bee1398e-7912-458a-9f8b-279b84a22566","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6825","nonce":2507226151}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6827","nonce":2507226151}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6831","nonce":2507226151}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6829","nonce":2507226151}]},"public_addr":"172.21.15.171:6825/2507226151","cluster_addr":"172.21.15.171:6827/2507226151","heartbeat_back_addr":"172.21.15.171:6831/2507226151","heartbeat_front_addr":"172.21.15.171:6829/2507226151","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:03.400774+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:28.117542+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:52.563066+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:17.387470+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:40.460850+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:05.110287+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:29.602919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:55.143996+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:6801/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:6800/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/67599134":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/4143435073":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6801/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/402721227":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/2398977550":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6800/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/1144243691":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/4047047180":"2024-04-27T04:34:42.908359+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:39:26.488 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-04-26T04:39:26.488 INFO:tasks.cephadm:Creating RBD pool 2024-04-26T04:39:26.488 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-04-26T04:39:26.499 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:26 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:26.139+0000 7f05f3c801c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:39:26.745 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:26.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:26 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/685173504' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:26.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:26 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/685173504' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:26.844 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:26 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:26.572+0000 7fac1fb5b1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:39:26.844 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:26 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:26.672+0000 7fac1fb5b1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:39:26.931 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:26 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/685173504' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:26.931 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:26 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:26.663+0000 7f05f3c801c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:39:26.932 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:26 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:26.755+0000 7f05f3c801c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:39:26.932 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:26 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:26.930+0000 7f05f3c801c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:39:27.128 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:26 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:26.868+0000 7fac1fb5b1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:39:27.456 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:27 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:27.192+0000 7fac1fb5b1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:39:27.456 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:27 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:27.454+0000 7fac1fb5b1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:39:27.473 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:27 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:27.217+0000 7f05f3c801c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:39:27.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:27 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:27.472+0000 7f05f3c801c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:39:27.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:27 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:27.705+0000 7f05f3c801c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:39:27.836 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:27 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:27.700+0000 7fac1fb5b1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:39:27.836 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:27 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:27.834+0000 7fac1fb5b1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:39:28.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:27 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:27.930+0000 7fac1fb5b1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:39:28.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:28 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:28.055+0000 7fac1fb5b1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:39:28.126 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:27 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:27.835+0000 7f05f3c801c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:39:28.126 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:27 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:27.926+0000 7f05f3c801c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:39:28.126 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:28 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:28.038+0000 7f05f3c801c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:39:28.127 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:28 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:28.125+0000 7f05f3c801c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:39:28.414 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:28 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:28.147+0000 7fac1fb5b1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:39:29.028 INFO:teuthology.orchestra.run.smithi156.stderr:pool 'rbd' created 2024-04-26T04:39:29.207 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:29 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2123134516' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:39:29.207 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:28 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:28.908+0000 7f05f3c801c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:39:29.208 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:29 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:29.099+0000 7f05f3c801c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:39:29.208 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:29 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: [26/Apr/2024:04:39:29] ENGINE Bus STARTING 2024-04-26T04:39:29.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:29 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2123134516' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:39:29.246 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:28 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:28.974+0000 7fac1fb5b1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:39:29.246 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:29 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:29.178+0000 7fac1fb5b1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:39:29.246 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:29 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2123134516' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:39:29.499 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:29 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: CherryPy Checker: 2024-04-26T04:39:29.499 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:29 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: The Application mounted at '' has an empty config. 2024-04-26T04:39:29.500 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:29 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: 2024-04-26T04:39:29.500 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:29 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: [26/Apr/2024:04:39:29] ENGINE Serving on http://:::9283 2024-04-26T04:39:29.500 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:29 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: [26/Apr/2024:04:39:29] ENGINE Bus STARTED 2024-04-26T04:39:29.593 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:29 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:39:29] ENGINE Bus STARTING 2024-04-26T04:39:29.593 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:29 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: CherryPy Checker: 2024-04-26T04:39:29.593 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:29 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: The Application mounted at '' has an empty config. 2024-04-26T04:39:29.594 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:29 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:39:29.594 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:29 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:39:29] ENGINE Serving on http://:::9283 2024-04-26T04:39:29.594 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:29 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:39:29] ENGINE Bus STARTED 2024-04-26T04:39:29.812 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-04-26T04:39:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2123134516' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-26T04:39:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T04:39:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: Standby manager daemon x restarted 2024-04-26T04:39:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: Standby manager daemon x started 2024-04-26T04:39:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/1336261381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:39:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/1336261381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:39:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/1336261381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:39:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/1336261381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:39:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: Active manager daemon y restarted 2024-04-26T04:39:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: Activating manager daemon y 2024-04-26T04:39:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T04:39:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: mgrmap e17: y(active, starting, since 0.00698245s), standbys: x 2024-04-26T04:39:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:39:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:39:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:39:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:39:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:39:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:39:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:39:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:39:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:39:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:39:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:39:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:39:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:39:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:39:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:39:30.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:39:30.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: Manager daemon y is now available 2024-04-26T04:39:30.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:30.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:30.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:30.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:30.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:39:30.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:39:30.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:39:30.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:39:30.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:30.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2123134516' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-26T04:39:30.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T04:39:30.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: Standby manager daemon x restarted 2024-04-26T04:39:30.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: Standby manager daemon x started 2024-04-26T04:39:30.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/1336261381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:39:30.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/1336261381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:39:30.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/1336261381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:39:30.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/1336261381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:39:30.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: Active manager daemon y restarted 2024-04-26T04:39:30.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: Activating manager daemon y 2024-04-26T04:39:30.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T04:39:30.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: mgrmap e17: y(active, starting, since 0.00698245s), standbys: x 2024-04-26T04:39:30.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:39:30.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:39:30.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:39:30.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:39:30.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:39:30.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:39:30.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:39:30.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:39:30.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:39:30.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:39:30.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:39:30.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:39:30.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:39:30.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:39:30.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:39:30.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:39:30.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: Manager daemon y is now available 2024-04-26T04:39:30.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:30.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:30.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:30.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:30.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:39:30.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:39:30.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:39:30.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:39:30.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:30 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:30.341 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2123134516' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-26T04:39:30.341 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T04:39:30.341 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: Standby manager daemon x restarted 2024-04-26T04:39:30.341 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: Standby manager daemon x started 2024-04-26T04:39:30.341 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/1336261381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:39:30.341 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/1336261381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:39:30.341 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/1336261381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:39:30.341 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/1336261381' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:39:30.341 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: Active manager daemon y restarted 2024-04-26T04:39:30.341 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: Activating manager daemon y 2024-04-26T04:39:30.341 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T04:39:30.341 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: mgrmap e17: y(active, starting, since 0.00698245s), standbys: x 2024-04-26T04:39:30.341 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:39:30.342 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:39:30.342 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:39:30.342 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:39:30.342 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:39:30.342 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:39:30.342 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:39:30.342 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:39:30.342 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:39:30.342 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:39:30.342 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:39:30.342 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:39:30.342 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:39:30.342 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:39:30.342 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:39:30.343 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:39:30.343 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: Manager daemon y is now available 2024-04-26T04:39:30.343 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:30.343 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:30.343 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:30.343 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:30.343 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:39:30.343 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:39:30.343 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:39:30.343 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:39:30.343 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:30 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:30.551 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:31.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:31 smithi171 ceph-mon[31139]: [26/Apr/2024:04:39:29] ENGINE Bus STARTING 2024-04-26T04:39:31.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:31 smithi171 ceph-mon[31139]: [26/Apr/2024:04:39:29] ENGINE Serving on https://172.21.15.156:7150 2024-04-26T04:39:31.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:31 smithi171 ceph-mon[31139]: [26/Apr/2024:04:39:29] ENGINE Bus STARTED 2024-04-26T04:39:31.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:31 smithi171 ceph-mon[31139]: mgrmap e18: y(active, since 1.01039s), standbys: x 2024-04-26T04:39:31.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:31 smithi171 ceph-mon[31139]: pgmap v4: 9 pgs: 2 creating+peering, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:31.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:31 smithi171 ceph-mon[31139]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T04:39:31.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:31 smithi156 ceph-mon[27570]: [26/Apr/2024:04:39:29] ENGINE Bus STARTING 2024-04-26T04:39:31.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:31 smithi156 ceph-mon[27570]: [26/Apr/2024:04:39:29] ENGINE Serving on https://172.21.15.156:7150 2024-04-26T04:39:31.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:31 smithi156 ceph-mon[27570]: [26/Apr/2024:04:39:29] ENGINE Bus STARTED 2024-04-26T04:39:31.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:31 smithi156 ceph-mon[27570]: mgrmap e18: y(active, since 1.01039s), standbys: x 2024-04-26T04:39:31.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:31 smithi156 ceph-mon[27570]: pgmap v4: 9 pgs: 2 creating+peering, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:31.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:31 smithi156 ceph-mon[27570]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T04:39:31.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:31 smithi156 ceph-mon[39130]: [26/Apr/2024:04:39:29] ENGINE Bus STARTING 2024-04-26T04:39:31.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:31 smithi156 ceph-mon[39130]: [26/Apr/2024:04:39:29] ENGINE Serving on https://172.21.15.156:7150 2024-04-26T04:39:31.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:31 smithi156 ceph-mon[39130]: [26/Apr/2024:04:39:29] ENGINE Bus STARTED 2024-04-26T04:39:31.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:31 smithi156 ceph-mon[39130]: mgrmap e18: y(active, since 1.01039s), standbys: x 2024-04-26T04:39:31.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:31 smithi156 ceph-mon[39130]: pgmap v4: 9 pgs: 2 creating+peering, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:31.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:31 smithi156 ceph-mon[39130]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T04:39:32.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[27570]: pgmap v5: 9 pgs: 2 creating+peering, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:32.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[27570]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:39:32.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[27570]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-26T04:39:32.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[27570]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T04:39:32.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:32.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:32.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:32.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:32.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:32.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[39130]: pgmap v5: 9 pgs: 2 creating+peering, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:32.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[39130]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:39:32.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[39130]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-26T04:39:32.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[39130]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T04:39:32.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:32.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:32.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:32.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:32.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:32 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:32.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:32 smithi171 ceph-mon[31139]: pgmap v5: 9 pgs: 2 creating+peering, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:32 smithi171 ceph-mon[31139]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:39:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:32 smithi171 ceph-mon[31139]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-26T04:39:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:32 smithi171 ceph-mon[31139]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T04:39:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:32 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:32 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:32 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:32 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:32 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:32 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:32 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:32 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:32 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:32 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:32 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:32 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:39:32.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:32 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:33.440 INFO:teuthology.orchestra.run.smithi156.stderr:enabled application 'rbd' on pool 'rbd' 2024-04-26T04:39:33.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:33 smithi171 ceph-mon[31139]: Adjusting osd_memory_target on smithi171 to 4016M 2024-04-26T04:39:33.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:33 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:33.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:33 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:33 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1232125640' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:39:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:33 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:33 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:33 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:33 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:39:33.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:33 smithi156 ceph-mon[27570]: Adjusting osd_memory_target on smithi171 to 4016M 2024-04-26T04:39:33.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:33 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:33.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:33 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:33.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:33 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1232125640' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:39:33.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:33 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:33.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:33 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:33.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:33 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:33.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:33 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:39:33.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:33 smithi156 ceph-mon[39130]: Adjusting osd_memory_target on smithi171 to 4016M 2024-04-26T04:39:33.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:33 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:33.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:33 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:33.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:33 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1232125640' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:39:33.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:33 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:33.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:33 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:33.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:33 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:33.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:33 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:39:34.291 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-26T04:39:34.292 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-26T04:39:34.292 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-26T04:39:34.556 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:34.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[27570]: Updating smithi156:/etc/ceph/ceph.conf 2024-04-26T04:39:34.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[27570]: Updating smithi171:/etc/ceph/ceph.conf 2024-04-26T04:39:34.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[27570]: pgmap v7: 9 pgs: 2 creating+peering, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:34.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[27570]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:39:34.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[27570]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:39:34.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1232125640' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:39:34.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[27570]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T04:39:34.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[27570]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:39:34.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[27570]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:39:34.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:34.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:34.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:34.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:34.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:34.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[27570]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-26T04:39:34.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[39130]: Updating smithi156:/etc/ceph/ceph.conf 2024-04-26T04:39:34.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[39130]: Updating smithi171:/etc/ceph/ceph.conf 2024-04-26T04:39:34.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[39130]: pgmap v7: 9 pgs: 2 creating+peering, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:34.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[39130]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:39:34.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[39130]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:39:34.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1232125640' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:39:34.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[39130]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T04:39:34.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[39130]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:39:34.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[39130]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:39:34.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:34.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:34.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:34.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:34.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:34.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:34 smithi156 ceph-mon[39130]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-26T04:39:34.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:39:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:39:34.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:34 smithi171 ceph-mon[31139]: Updating smithi156:/etc/ceph/ceph.conf 2024-04-26T04:39:34.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:34 smithi171 ceph-mon[31139]: Updating smithi171:/etc/ceph/ceph.conf 2024-04-26T04:39:34.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:34 smithi171 ceph-mon[31139]: pgmap v7: 9 pgs: 2 creating+peering, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:34.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:34 smithi171 ceph-mon[31139]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:39:34.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:34 smithi171 ceph-mon[31139]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:39:34.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:34 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1232125640' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:39:34.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:34 smithi171 ceph-mon[31139]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T04:39:34.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:34 smithi171 ceph-mon[31139]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:39:34.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:34 smithi171 ceph-mon[31139]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:39:34.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:34 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:34.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:34 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:34.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:34 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:34.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:34 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:34.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:34 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:34.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:34 smithi171 ceph-mon[31139]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-26T04:39:35.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:35 smithi171 ceph-mon[31139]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:39:35.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:35 smithi171 ceph-mon[31139]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:39:35.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:35 smithi171 ceph-mon[31139]: Deploying daemon alertmanager.a on smithi156 2024-04-26T04:39:35.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:35 smithi156 ceph-mon[27570]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:39:35.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:35 smithi156 ceph-mon[27570]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:39:35.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:35 smithi156 ceph-mon[27570]: Deploying daemon alertmanager.a on smithi156 2024-04-26T04:39:35.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:35 smithi156 ceph-mon[39130]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:39:35.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:35 smithi156 ceph-mon[39130]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:39:35.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:35 smithi156 ceph-mon[39130]: Deploying daemon alertmanager.a on smithi156 2024-04-26T04:39:36.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:36 smithi156 ceph-mon[27570]: pgmap v9: 9 pgs: 2 creating+peering, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:36.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:36 smithi156 ceph-mon[27570]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:39:36.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:36 smithi156 ceph-mon[27570]: Cluster is now healthy 2024-04-26T04:39:36.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:36 smithi156 ceph-mon[39130]: pgmap v9: 9 pgs: 2 creating+peering, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:36.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:36 smithi156 ceph-mon[39130]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:39:36.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:36 smithi156 ceph-mon[39130]: Cluster is now healthy 2024-04-26T04:39:36.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:36 smithi171 ceph-mon[31139]: pgmap v9: 9 pgs: 2 creating+peering, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:36.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:36 smithi171 ceph-mon[31139]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:39:36.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:36 smithi171 ceph-mon[31139]: Cluster is now healthy 2024-04-26T04:39:37.035 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:39:37.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:37 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1609459091' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:39:37.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:37 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1609459091' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:39:37.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:37 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1609459091' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:39:38.594 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":20,"flags":0,"active_gid":24344,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6800","nonce":367046062},{"type":"v1","addr":"172.21.15.156:6801","nonce":367046062}]},"active_addr":"172.21.15.156:6801/367046062","active_change":"2024-04-26T04:39:29.182748+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24331,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.156:8443/","prometheus":"http://172.21.15.156:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":3291954384}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":32257782}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":1361591211}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":1772897570}]}]} 2024-04-26T04:39:38.599 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-26T04:39:38.599 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-26T04:39:38.599 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:39:38.744 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:38 smithi156 ceph-mon[27570]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:39:38.745 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:38 smithi156 ceph-mon[39130]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:39:38.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:38 smithi171 ceph-mon[31139]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:39:38.852 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:40.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:40 smithi171 ceph-mon[31139]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-26T04:39:40.839 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:40 smithi156 ceph-mon[27570]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-26T04:39:40.839 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:40 smithi156 ceph-mon[39130]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-26T04:39:42.543 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:42 smithi156 ceph-mon[27570]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:39:42.543 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:42 smithi156 ceph-mon[39130]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:39:42.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:42 smithi171 ceph-mon[31139]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:39:42.853 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:39:42.853 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":63,"fsid":"110027f2-0386-11ef-bc93-c7b262605968","created":"2024-04-26T04:33:46.638460+0000","modified":"2024-04-26T04:39:33.433103+0000","last_up_change":"2024-04-26T04:38:56.608997+0000","last_in_change":"2024-04-26T04:38:39.401044+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:36:55.213219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:39:28.969228+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"6b59fe49-0273-4115-91d9-6faea7d37616","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6803","nonce":3436276177}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6805","nonce":3436276177}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6809","nonce":3436276177}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6807","nonce":3436276177}]},"public_addr":"172.21.15.156:6803/3436276177","cluster_addr":"172.21.15.156:6805/3436276177","heartbeat_back_addr":"172.21.15.156:6809/3436276177","heartbeat_front_addr":"172.21.15.156:6807/3436276177","state":["exists","up"]},{"osd":1,"uuid":"c9673007-264f-4d7c-8234-bfd514e70d2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6811","nonce":273749030}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6813","nonce":273749030}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6817","nonce":273749030}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6815","nonce":273749030}]},"public_addr":"172.21.15.156:6811/273749030","cluster_addr":"172.21.15.156:6813/273749030","heartbeat_back_addr":"172.21.15.156:6817/273749030","heartbeat_front_addr":"172.21.15.156:6815/273749030","state":["exists","up"]},{"osd":2,"uuid":"d0051bbe-c0cd-42f4-9d8c-6b0a3e71886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6819","nonce":3653838089}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6821","nonce":3653838089}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6825","nonce":3653838089}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6823","nonce":3653838089}]},"public_addr":"172.21.15.156:6819/3653838089","cluster_addr":"172.21.15.156:6821/3653838089","heartbeat_back_addr":"172.21.15.156:6825/3653838089","heartbeat_front_addr":"172.21.15.156:6823/3653838089","state":["exists","up"]},{"osd":3,"uuid":"479e109c-1e4a-4124-9442-f85d63df6d04","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6827","nonce":2175305841}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6829","nonce":2175305841}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6832","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6833","nonce":2175305841}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6831","nonce":2175305841}]},"public_addr":"172.21.15.156:6827/2175305841","cluster_addr":"172.21.15.156:6829/2175305841","heartbeat_back_addr":"172.21.15.156:6833/2175305841","heartbeat_front_addr":"172.21.15.156:6831/2175305841","state":["exists","up"]},{"osd":4,"uuid":"b090dbe4-ed23-4015-a864-074aa64cace4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6801","nonce":2255415442}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6803","nonce":2255415442}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6807","nonce":2255415442}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6805","nonce":2255415442}]},"public_addr":"172.21.15.171:6801/2255415442","cluster_addr":"172.21.15.171:6803/2255415442","heartbeat_back_addr":"172.21.15.171:6807/2255415442","heartbeat_front_addr":"172.21.15.171:6805/2255415442","state":["exists","up"]},{"osd":5,"uuid":"f506481f-428b-40d6-a54b-8ef411672b53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6809","nonce":2373798416}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6811","nonce":2373798416}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6815","nonce":2373798416}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6813","nonce":2373798416}]},"public_addr":"172.21.15.171:6809/2373798416","cluster_addr":"172.21.15.171:6811/2373798416","heartbeat_back_addr":"172.21.15.171:6815/2373798416","heartbeat_front_addr":"172.21.15.171:6813/2373798416","state":["exists","up"]},{"osd":6,"uuid":"f1fd019a-372d-4b4e-bede-7659038df10a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6817","nonce":2328644410}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6819","nonce":2328644410}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6823","nonce":2328644410}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6821","nonce":2328644410}]},"public_addr":"172.21.15.171:6817/2328644410","cluster_addr":"172.21.15.171:6819/2328644410","heartbeat_back_addr":"172.21.15.171:6823/2328644410","heartbeat_front_addr":"172.21.15.171:6821/2328644410","state":["exists","up"]},{"osd":7,"uuid":"bee1398e-7912-458a-9f8b-279b84a22566","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6825","nonce":2507226151}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6827","nonce":2507226151}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6831","nonce":2507226151}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6829","nonce":2507226151}]},"public_addr":"172.21.15.171:6825/2507226151","cluster_addr":"172.21.15.171:6827/2507226151","heartbeat_back_addr":"172.21.15.171:6831/2507226151","heartbeat_front_addr":"172.21.15.171:6829/2507226151","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:03.400774+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:28.117542+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:52.563066+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:17.387470+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:40.460850+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:05.110287+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:29.602919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:55.143996+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:0/3755168891":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/3331144037":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/711432262":"2024-04-27T04:39:29.182492+0000","172.21.15.156:6801/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:6800/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/67599134":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/4143435073":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6800/1084840701":"2024-04-27T04:39:29.182492+0000","172.21.15.156:6801/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/402721227":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/2398977550":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6800/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6801/1084840701":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/314056408":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/1144243691":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/4047047180":"2024-04-27T04:34:42.908359+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:39:43.708 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:43 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3975513002' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:43.709 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:43 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3975513002' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:43.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:43 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3975513002' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:43.994 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:43 smithi156 systemd[1]: Starting Ceph alertmanager.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:39:44.278 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-26T04:39:44.279 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:39:44.551 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:44 smithi156 ceph-mon[27570]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:39:44.551 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:44 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:44.551 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:44 smithi156 ceph-mon[39130]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:39:44.551 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:44 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:44.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:44 smithi171 ceph-mon[31139]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:39:44.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:44 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:44.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:44 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:39:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:39:44.836 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:44 smithi156 podman[70383]: 2024-04-26 04:39:44.452704444 +0000 UTC m=+0.025258292 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-26T04:39:44.836 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:44 smithi156 podman[70383]: 2024-04-26 04:39:44.561105604 +0000 UTC m=+0.133659480 volume create 2571df10f3e87aa4b42a444049676ca7e5e4d6c83f97eca9db8fdbd4bef4e987 2024-04-26T04:39:44.837 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:44 smithi156 podman[70383]: 2024-04-26 04:39:44.644946651 +0000 UTC m=+0.217500485 container create 0dea4b285c05366b14edace4597f7785d7083ffe0a03e53c1fc64e2073ccf8a6 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:45.112 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:45.236 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:44 smithi156 podman[70383]: 2024-04-26 04:39:44.939310621 +0000 UTC m=+0.511864464 container init 0dea4b285c05366b14edace4597f7785d7083ffe0a03e53c1fc64e2073ccf8a6 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:45.236 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:44 smithi156 podman[70383]: 2024-04-26 04:39:44.944930613 +0000 UTC m=+0.517484448 container start 0dea4b285c05366b14edace4597f7785d7083ffe0a03e53c1fc64e2073ccf8a6 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:45.236 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[70425]: ts=2024-04-26T04:39:44.957Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-26T04:39:45.236 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[70425]: ts=2024-04-26T04:39:44.957Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-26T04:39:45.237 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[70425]: ts=2024-04-26T04:39:44.958Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.156 port=9094 2024-04-26T04:39:45.237 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[70425]: ts=2024-04-26T04:39:44.959Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T04:39:45.237 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[70425]: ts=2024-04-26T04:39:44.982Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:39:45.237 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[70425]: ts=2024-04-26T04:39:44.982Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:39:45.237 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[70425]: ts=2024-04-26T04:39:44.985Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-26T04:39:45.237 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[70425]: ts=2024-04-26T04:39:44.985Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-26T04:39:45.237 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:45 smithi156 bash[70383]: 0dea4b285c05366b14edace4597f7785d7083ffe0a03e53c1fc64e2073ccf8a6 2024-04-26T04:39:45.237 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:45 smithi156 systemd[1]: Started Ceph alertmanager.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:39:45.578 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:45 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:39:45] ENGINE Bus STOPPING 2024-04-26T04:39:45.578 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:45 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:39:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:39:45.578 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:45 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:39:45] ENGINE Bus STOPPED 2024-04-26T04:39:45.579 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:45 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:39:45] ENGINE Bus STARTING 2024-04-26T04:39:45.579 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:45 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:39:45] ENGINE Serving on http://:::9283 2024-04-26T04:39:45.579 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:39:45 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:39:45] ENGINE Bus STARTED 2024-04-26T04:39:46.491 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.491 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.491 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.491 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[27570]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:39:46.491 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.491 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.492 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.492 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:39:46.492 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[27570]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:39:46.492 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.492 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[27570]: Deploying daemon grafana.a on smithi171 2024-04-26T04:39:46.492 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.492 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.492 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.492 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[39130]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:39:46.492 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.492 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.492 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.493 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:39:46.493 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[39130]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:39:46.493 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.493 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-mon[39130]: Deploying daemon grafana.a on smithi171 2024-04-26T04:39:46.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:46 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:46 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:46 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:46 smithi171 ceph-mon[31139]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:39:46.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:46 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:46 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:46 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:46 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:39:46.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:46 smithi171 ceph-mon[31139]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:39:46.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:46 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:46.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:46 smithi171 ceph-mon[31139]: Deploying daemon grafana.a on smithi171 2024-04-26T04:39:47.093 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:46 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[70425]: ts=2024-04-26T04:39:46.959Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000495281s 2024-04-26T04:39:47.215 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:39:47.215 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":63,"fsid":"110027f2-0386-11ef-bc93-c7b262605968","created":"2024-04-26T04:33:46.638460+0000","modified":"2024-04-26T04:39:33.433103+0000","last_up_change":"2024-04-26T04:38:56.608997+0000","last_in_change":"2024-04-26T04:38:39.401044+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:36:55.213219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:39:28.969228+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"6b59fe49-0273-4115-91d9-6faea7d37616","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6803","nonce":3436276177}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6805","nonce":3436276177}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6809","nonce":3436276177}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6807","nonce":3436276177}]},"public_addr":"172.21.15.156:6803/3436276177","cluster_addr":"172.21.15.156:6805/3436276177","heartbeat_back_addr":"172.21.15.156:6809/3436276177","heartbeat_front_addr":"172.21.15.156:6807/3436276177","state":["exists","up"]},{"osd":1,"uuid":"c9673007-264f-4d7c-8234-bfd514e70d2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6811","nonce":273749030}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6813","nonce":273749030}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6817","nonce":273749030}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6815","nonce":273749030}]},"public_addr":"172.21.15.156:6811/273749030","cluster_addr":"172.21.15.156:6813/273749030","heartbeat_back_addr":"172.21.15.156:6817/273749030","heartbeat_front_addr":"172.21.15.156:6815/273749030","state":["exists","up"]},{"osd":2,"uuid":"d0051bbe-c0cd-42f4-9d8c-6b0a3e71886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6819","nonce":3653838089}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6821","nonce":3653838089}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6825","nonce":3653838089}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6823","nonce":3653838089}]},"public_addr":"172.21.15.156:6819/3653838089","cluster_addr":"172.21.15.156:6821/3653838089","heartbeat_back_addr":"172.21.15.156:6825/3653838089","heartbeat_front_addr":"172.21.15.156:6823/3653838089","state":["exists","up"]},{"osd":3,"uuid":"479e109c-1e4a-4124-9442-f85d63df6d04","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6827","nonce":2175305841}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6829","nonce":2175305841}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6832","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6833","nonce":2175305841}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6831","nonce":2175305841}]},"public_addr":"172.21.15.156:6827/2175305841","cluster_addr":"172.21.15.156:6829/2175305841","heartbeat_back_addr":"172.21.15.156:6833/2175305841","heartbeat_front_addr":"172.21.15.156:6831/2175305841","state":["exists","up"]},{"osd":4,"uuid":"b090dbe4-ed23-4015-a864-074aa64cace4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6801","nonce":2255415442}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6803","nonce":2255415442}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6807","nonce":2255415442}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6805","nonce":2255415442}]},"public_addr":"172.21.15.171:6801/2255415442","cluster_addr":"172.21.15.171:6803/2255415442","heartbeat_back_addr":"172.21.15.171:6807/2255415442","heartbeat_front_addr":"172.21.15.171:6805/2255415442","state":["exists","up"]},{"osd":5,"uuid":"f506481f-428b-40d6-a54b-8ef411672b53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6809","nonce":2373798416}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6811","nonce":2373798416}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6815","nonce":2373798416}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6813","nonce":2373798416}]},"public_addr":"172.21.15.171:6809/2373798416","cluster_addr":"172.21.15.171:6811/2373798416","heartbeat_back_addr":"172.21.15.171:6815/2373798416","heartbeat_front_addr":"172.21.15.171:6813/2373798416","state":["exists","up"]},{"osd":6,"uuid":"f1fd019a-372d-4b4e-bede-7659038df10a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6817","nonce":2328644410}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6819","nonce":2328644410}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6823","nonce":2328644410}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6821","nonce":2328644410}]},"public_addr":"172.21.15.171:6817/2328644410","cluster_addr":"172.21.15.171:6819/2328644410","heartbeat_back_addr":"172.21.15.171:6823/2328644410","heartbeat_front_addr":"172.21.15.171:6821/2328644410","state":["exists","up"]},{"osd":7,"uuid":"bee1398e-7912-458a-9f8b-279b84a22566","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6825","nonce":2507226151}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6827","nonce":2507226151}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6831","nonce":2507226151}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6829","nonce":2507226151}]},"public_addr":"172.21.15.171:6825/2507226151","cluster_addr":"172.21.15.171:6827/2507226151","heartbeat_back_addr":"172.21.15.171:6831/2507226151","heartbeat_front_addr":"172.21.15.171:6829/2507226151","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:03.400774+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:28.117542+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:52.563066+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:17.387470+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:40.460850+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:05.110287+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:29.602919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:55.143996+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:0/3755168891":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/3331144037":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/711432262":"2024-04-27T04:39:29.182492+0000","172.21.15.156:6801/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:6800/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/67599134":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/4143435073":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6800/1084840701":"2024-04-27T04:39:29.182492+0000","172.21.15.156:6801/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/402721227":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/2398977550":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6800/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6801/1084840701":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/314056408":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/1144243691":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/4047047180":"2024-04-27T04:34:42.908359+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:39:47.498 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:47 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1286971416' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:47.498 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:47 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1286971416' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:47.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:47 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1286971416' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:47.917 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-26T04:39:47.917 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-26T04:39:47.918 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-26T04:39:47.918 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-26T04:39:47.919 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-26T04:39:47.919 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-26T04:39:47.919 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-26T04:39:47.920 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-26T04:39:48.491 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:48.494 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:48.495 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:48.495 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:48.496 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:48.496 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:48.500 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:48.501 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:48.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:48 smithi156 ceph-mon[27570]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:39:48.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:48 smithi156 ceph-mon[39130]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:39:48.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:48 smithi171 ceph-mon[31139]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:39:50.426 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:50 smithi156 ceph-mon[39130]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:50.426 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:50 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:50.426 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:50 smithi156 ceph-mon[27570]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:50.426 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:50 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:50.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:50 smithi171 ceph-mon[31139]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:50.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:50 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:39:52.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:52 smithi171 ceph-mon[31139]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:52.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:52 smithi156 ceph-mon[39130]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:52.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:52 smithi156 ceph-mon[27570]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:54.548 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:54 smithi156 ceph-mon[27570]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:54.548 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:54 smithi156 ceph-mon[39130]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:54.713 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:54 smithi171 ceph-mon[31139]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:54.713 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:39:54 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:39:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:39:55.343 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:39:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[70425]: ts=2024-04-26T04:39:54.963Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.00369104s 2024-04-26T04:39:56.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:56 smithi156 ceph-mon[39130]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:56.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:56 smithi156 ceph-mon[27570]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:56.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:56 smithi171 ceph-mon[31139]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:58.398 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:58 smithi156 ceph-mon[39130]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:58.398 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:58 smithi156 ceph-mon[27570]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:58.591 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:58 smithi171 ceph-mon[31139]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:59.311 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:39:59 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:59.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:39:59 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:59.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:39:59 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:59.758 INFO:teuthology.orchestra.run.smithi156.stdout:107374182434 2024-04-26T04:39:59.758 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-26T04:39:59.999 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:39:59 smithi171 podman[62291]: 2024-04-26 04:39:59.937501166 +0000 UTC m=+0.132225299 container create d42cfd971c1c00d9d339c7dfa97b5902451a2466958c5567cf5abecd0c1112a0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, version=8.5, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, com.redhat.component=ubi8-container, vcs-type=git, description=Ceph Grafana Container, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-26T04:40:00.284 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-mon[31139]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:00.285 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-mon[31139]: overall HEALTH_OK 2024-04-26T04:40:00.285 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 podman[62291]: 2024-04-26 04:40:00.12275426 +0000 UTC m=+0.317478395 container init d42cfd971c1c00d9d339c7dfa97b5902451a2466958c5567cf5abecd0c1112a0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.buildah.version=1.32.0, version=8.5, description=Ceph Grafana Container, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, distribution-scope=public, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc.) 2024-04-26T04:40:00.285 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 podman[62291]: 2024-04-26 04:40:00.127977829 +0000 UTC m=+0.322701963 container start d42cfd971c1c00d9d339c7dfa97b5902451a2466958c5567cf5abecd0c1112a0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, description=Ceph Grafana Container, distribution-scope=public, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., vcs-type=git, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, name=ubi8, architecture=x86_64, build-date=2022-04-29T11:48:11.055897) 2024-04-26T04:40:00.285 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 bash[62291]: d42cfd971c1c00d9d339c7dfa97b5902451a2466958c5567cf5abecd0c1112a0 2024-04-26T04:40:00.285 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 systemd[1]: Started Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:40:00.536 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=settings t=2024-04-26T04:40:00.284356838Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-04-26T04:40:00.537 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=settings t=2024-04-26T04:40:00.284596366Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-04-26T04:40:00.537 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=settings t=2024-04-26T04:40:00.284673735Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-04-26T04:40:00.537 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=settings t=2024-04-26T04:40:00.284689733Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-04-26T04:40:00.537 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=settings t=2024-04-26T04:40:00.28470303Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-26T04:40:00.537 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=settings t=2024-04-26T04:40:00.284715163Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-26T04:40:00.537 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=settings t=2024-04-26T04:40:00.284726151Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-26T04:40:00.537 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=settings t=2024-04-26T04:40:00.284737009Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-26T04:40:00.537 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=settings t=2024-04-26T04:40:00.284748369Z level=info msg="Path Home" path=/usr/share/grafana 2024-04-26T04:40:00.537 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=settings t=2024-04-26T04:40:00.284759847Z level=info msg="Path Data" path=/var/lib/grafana 2024-04-26T04:40:00.537 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=settings t=2024-04-26T04:40:00.28477095Z level=info msg="Path Logs" path=/var/log/grafana 2024-04-26T04:40:00.537 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=settings t=2024-04-26T04:40:00.284781588Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-04-26T04:40:00.538 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=settings t=2024-04-26T04:40:00.284792017Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-04-26T04:40:00.538 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=settings t=2024-04-26T04:40:00.284802787Z level=info msg="App mode production" 2024-04-26T04:40:00.538 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=sqlstore t=2024-04-26T04:40:00.284934684Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-04-26T04:40:00.538 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=sqlstore t=2024-04-26T04:40:00.284968243Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-26T04:40:00.538 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.285912698Z level=info msg="Starting DB migrations" 2024-04-26T04:40:00.538 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.286002398Z level=info msg="Executing migration" id="create migration_log table" 2024-04-26T04:40:00.538 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.286555578Z level=info msg="Executing migration" id="create user table" 2024-04-26T04:40:00.538 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.287030869Z level=info msg="Executing migration" id="add unique index user.login" 2024-04-26T04:40:00.538 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.289848867Z level=info msg="Executing migration" id="add unique index user.email" 2024-04-26T04:40:00.538 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.292570542Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-04-26T04:40:00.539 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.29536399Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-04-26T04:40:00.539 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.298085506Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-04-26T04:40:00.539 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.300653106Z level=info msg="Executing migration" id="create user table v2" 2024-04-26T04:40:00.539 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.301108845Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-04-26T04:40:00.539 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.303984328Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-04-26T04:40:00.539 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.304509548Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-04-26T04:40:00.539 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.306760458Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-04-26T04:40:00.539 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.309341242Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-04-26T04:40:00.539 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.312139637Z level=info msg="Executing migration" id="Update user table charset" 2024-04-26T04:40:00.539 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.314705917Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-04-26T04:40:00.539 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.31735874Z level=info msg="Executing migration" id="Add missing user data" 2024-04-26T04:40:00.539 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.320094169Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-04-26T04:40:00.539 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.322948038Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-04-26T04:40:00.539 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.325633023Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-04-26T04:40:00.539 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.328277652Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-04-26T04:40:00.540 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.331073382Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-04-26T04:40:00.541 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.333866118Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-04-26T04:40:00.541 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.336401829Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-04-26T04:40:00.541 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.338913662Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-04-26T04:40:00.541 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.339394647Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-04-26T04:40:00.541 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.339844697Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-04-26T04:40:00.541 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.342453428Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-04-26T04:40:00.541 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.34513945Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-04-26T04:40:00.541 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.345549932Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-04-26T04:40:00.542 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.345951629Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-04-26T04:40:00.542 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.34633863Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-26T04:40:00.542 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.346999332Z level=info msg="Executing migration" id="create temp_user v2" 2024-04-26T04:40:00.542 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.350503624Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-04-26T04:40:00.542 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.351072943Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-04-26T04:40:00.542 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.351471198Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-04-26T04:40:00.542 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.351888754Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-04-26T04:40:00.542 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.352311321Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-04-26T04:40:00.542 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.352645278Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-04-26T04:40:00.542 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.353024416Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-04-26T04:40:00.542 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.353366215Z level=info msg="Executing migration" id="create star table" 2024-04-26T04:40:00.542 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.353724047Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-04-26T04:40:00.542 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.354201722Z level=info msg="Executing migration" id="create org table v1" 2024-04-26T04:40:00.543 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.35462855Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-04-26T04:40:00.543 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.355067514Z level=info msg="Executing migration" id="create org_user table v1" 2024-04-26T04:40:00.543 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.355460501Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-04-26T04:40:00.543 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.355894694Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-04-26T04:40:00.543 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.356331873Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-04-26T04:40:00.543 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.356741784Z level=info msg="Executing migration" id="Update org table charset" 2024-04-26T04:40:00.543 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.357060857Z level=info msg="Executing migration" id="Update org_user table charset" 2024-04-26T04:40:00.543 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.357350449Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-04-26T04:40:00.543 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.357655926Z level=info msg="Executing migration" id="create dashboard table" 2024-04-26T04:40:00.543 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.358082071Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-04-26T04:40:00.543 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.358507026Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-04-26T04:40:00.543 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.358938134Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-04-26T04:40:00.543 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.359338513Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-04-26T04:40:00.543 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.362081566Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-26T04:40:00.544 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.364902177Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-04-26T04:40:00.544 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.367699968Z level=info msg="Executing migration" id="create dashboard v2" 2024-04-26T04:40:00.544 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.370472881Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-04-26T04:40:00.544 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.373132007Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-04-26T04:40:00.544 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.375960552Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-04-26T04:40:00.544 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.378699055Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-04-26T04:40:00.544 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.379130087Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-04-26T04:40:00.544 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.379413628Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-04-26T04:40:00.544 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.381984689Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-04-26T04:40:00.544 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.384518232Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-04-26T04:40:00.544 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.387234923Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-04-26T04:40:00.544 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.389925716Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-04-26T04:40:00.544 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.392725466Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-04-26T04:40:00.544 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.39543413Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-04-26T04:40:00.545 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.398040839Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-04-26T04:40:00.545 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.398411026Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-04-26T04:40:00.545 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.39869036Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-04-26T04:40:00.545 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.400868959Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-04-26T04:40:00.545 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.403443422Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-04-26T04:40:00.545 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.406177494Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-04-26T04:40:00.545 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.408776237Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-04-26T04:40:00.545 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.411450939Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-04-26T04:40:00.545 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.413895144Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-04-26T04:40:00.545 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.416807531Z level=info msg="Executing migration" id="Update dashboard title length" 2024-04-26T04:40:00.545 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.419346397Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-26T04:40:00.545 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.422013253Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-04-26T04:40:00.545 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.424852594Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-26T04:40:00.545 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.427508926Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-04-26T04:40:00.546 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.42817826Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-26T04:40:00.546 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.429149427Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-26T04:40:00.546 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.429846363Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-04-26T04:40:00.546 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.430837241Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-04-26T04:40:00.546 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.431574952Z level=info msg="Executing migration" id="Add check_sum column" 2024-04-26T04:40:00.546 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.432397318Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-04-26T04:40:00.546 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.433362762Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-04-26T04:40:00.546 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.434052186Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-04-26T04:40:00.546 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.434892539Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-04-26T04:40:00.546 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.435825925Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-04-26T04:40:00.546 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.436584099Z level=info msg="Executing migration" id="create data_source table" 2024-04-26T04:40:00.546 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.437570247Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-04-26T04:40:00.546 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.438155033Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-04-26T04:40:00.547 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.438599699Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-04-26T04:40:00.547 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.439385053Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-04-26T04:40:00.547 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.44017859Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-04-26T04:40:00.547 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.441183738Z level=info msg="Executing migration" id="create data_source table v2" 2024-04-26T04:40:00.547 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.442077906Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-04-26T04:40:00.547 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.442910618Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-04-26T04:40:00.547 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.443790684Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-04-26T04:40:00.547 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.444671145Z level=info msg="Executing migration" id="Add column with_credentials" 2024-04-26T04:40:00.547 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.445558671Z level=info msg="Executing migration" id="Add secure json data column" 2024-04-26T04:40:00.547 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.446396866Z level=info msg="Executing migration" id="Update data_source table charset" 2024-04-26T04:40:00.547 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.447255769Z level=info msg="Executing migration" id="Update initial version to 1" 2024-04-26T04:40:00.548 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.448112944Z level=info msg="Executing migration" id="Add read_only data column" 2024-04-26T04:40:00.548 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.44899628Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-04-26T04:40:00.548 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.449897641Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-04-26T04:40:00.548 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.450813866Z level=info msg="Executing migration" id="Add uid column" 2024-04-26T04:40:00.548 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.451618153Z level=info msg="Executing migration" id="Update uid value" 2024-04-26T04:40:00.548 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.452466839Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-04-26T04:40:00.548 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.453407681Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-04-26T04:40:00.548 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.454338528Z level=info msg="Executing migration" id="create api_key table" 2024-04-26T04:40:00.548 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.455059948Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-04-26T04:40:00.548 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.455905094Z level=info msg="Executing migration" id="add index api_key.key" 2024-04-26T04:40:00.548 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.45676554Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-04-26T04:40:00.548 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.457635726Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-04-26T04:40:00.548 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.46035816Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-04-26T04:40:00.548 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.463097929Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-04-26T04:40:00.549 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.465938515Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-04-26T04:40:00.549 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.468829857Z level=info msg="Executing migration" id="create api_key table v2" 2024-04-26T04:40:00.549 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.469227188Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-04-26T04:40:00.549 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.471923145Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-04-26T04:40:00.549 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.47239381Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-04-26T04:40:00.549 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.474798722Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-04-26T04:40:00.549 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.475122757Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-04-26T04:40:00.549 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.477419166Z level=info msg="Executing migration" id="Update api_key table charset" 2024-04-26T04:40:00.549 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.47775226Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-04-26T04:40:00.549 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.480265315Z level=info msg="Executing migration" id="Add service account foreign key" 2024-04-26T04:40:00.550 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.482966756Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-04-26T04:40:00.550 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.485685213Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-04-26T04:40:00.550 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.486260188Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-04-26T04:40:00.550 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.488913969Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-04-26T04:40:00.550 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.489308532Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-04-26T04:40:00.550 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.491699174Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-04-26T04:40:00.550 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.492102243Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-04-26T04:40:00.550 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.494643536Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-26T04:40:00.550 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.495173054Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-26T04:40:00.551 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.498114316Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-04-26T04:40:00.551 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.500838092Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-04-26T04:40:00.551 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.503295142Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-04-26T04:40:00.551 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.503910826Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-04-26T04:40:00.551 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.506763482Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-26T04:40:00.551 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.509319601Z level=info msg="Executing migration" id="create quota table v1" 2024-04-26T04:40:00.551 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.512052944Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-26T04:40:00.551 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.512485017Z level=info msg="Executing migration" id="Update quota table charset" 2024-04-26T04:40:00.551 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.512753533Z level=info msg="Executing migration" id="create plugin_setting table" 2024-04-26T04:40:00.551 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.51519851Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-26T04:40:00.551 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.517810282Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-04-26T04:40:00.551 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.520463116Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-04-26T04:40:00.551 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.520782126Z level=info msg="Executing migration" id="create session table" 2024-04-26T04:40:00.551 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.521230751Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-04-26T04:40:00.552 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.521537013Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-04-26T04:40:00.552 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.523890416Z level=info msg="Executing migration" id="create playlist table v2" 2024-04-26T04:40:00.552 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.524313473Z level=info msg="Executing migration" id="create playlist item table v2" 2024-04-26T04:40:00.552 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.524721058Z level=info msg="Executing migration" id="Update playlist table charset" 2024-04-26T04:40:00.552 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.527473159Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-04-26T04:40:00.552 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.530139195Z level=info msg="Executing migration" id="drop preferences table v2" 2024-04-26T04:40:00.552 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.532927409Z level=info msg="Executing migration" id="drop preferences table v3" 2024-04-26T04:40:00.552 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.533286959Z level=info msg="Executing migration" id="create preferences table v3" 2024-04-26T04:40:00.552 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.536064521Z level=info msg="Executing migration" id="Update preferences table charset" 2024-04-26T04:40:00.552 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.538749185Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-04-26T04:40:00.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:00 smithi156 ceph-mon[39130]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:00.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:00 smithi156 ceph-mon[39130]: overall HEALTH_OK 2024-04-26T04:40:00.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:00 smithi156 ceph-mon[27570]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:00.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:00 smithi156 ceph-mon[27570]: overall HEALTH_OK 2024-04-26T04:40:00.743 INFO:teuthology.orchestra.run.smithi156.stdout:197568495635 2024-04-26T04:40:00.743 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-26T04:40:00.789 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.541433432Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-04-26T04:40:00.789 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.541776967Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-04-26T04:40:00.789 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.544520565Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-04-26T04:40:00.789 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.547237999Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-04-26T04:40:00.789 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.549932271Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-04-26T04:40:00.789 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.550411567Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-04-26T04:40:00.789 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.55136411Z level=info msg="Executing migration" id="create alert table v1" 2024-04-26T04:40:00.789 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.551988852Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-04-26T04:40:00.790 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.552897833Z level=info msg="Executing migration" id="add index alert state" 2024-04-26T04:40:00.790 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.553419764Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-04-26T04:40:00.790 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.554325393Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-04-26T04:40:00.790 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.555027979Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-26T04:40:00.790 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.555864686Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-26T04:40:00.790 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.556285461Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-26T04:40:00.790 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.557683468Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-04-26T04:40:00.790 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.558497805Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-26T04:40:00.790 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.55939287Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-04-26T04:40:00.790 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.559834261Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-04-26T04:40:00.790 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.560742579Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-04-26T04:40:00.790 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.561677622Z level=info msg="Executing migration" id="Add column is_default" 2024-04-26T04:40:00.791 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.562363212Z level=info msg="Executing migration" id="Add column frequency" 2024-04-26T04:40:00.791 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.563028856Z level=info msg="Executing migration" id="Add column send_reminder" 2024-04-26T04:40:00.791 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.56388118Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-04-26T04:40:00.791 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.564884455Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-04-26T04:40:00.791 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.56554576Z level=info msg="Executing migration" id="Update alert table charset" 2024-04-26T04:40:00.791 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.565861174Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-04-26T04:40:00.791 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.566822419Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-04-26T04:40:00.791 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.567765943Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-26T04:40:00.791 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.568495817Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-04-26T04:40:00.791 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.568916373Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-04-26T04:40:00.791 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.569930392Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-26T04:40:00.791 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.570602931Z level=info msg="Executing migration" id="Add for to alert table" 2024-04-26T04:40:00.791 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.571447935Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-04-26T04:40:00.791 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.572120197Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-04-26T04:40:00.792 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.57301351Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-04-26T04:40:00.792 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.57375857Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-04-26T04:40:00.792 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.5746972Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-04-26T04:40:00.792 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.575333977Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-04-26T04:40:00.793 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.576263599Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-04-26T04:40:00.793 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.576886988Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-04-26T04:40:00.794 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.577343756Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-04-26T04:40:00.794 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.580232275Z level=info msg="Executing migration" id="create annotation table v5" 2024-04-26T04:40:00.794 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.580940233Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-04-26T04:40:00.794 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.58140146Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-04-26T04:40:00.794 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.584352408Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-04-26T04:40:00.794 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.58685489Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-04-26T04:40:00.794 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.589707968Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-04-26T04:40:00.794 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.590257111Z level=info msg="Executing migration" id="Update annotation table charset" 2024-04-26T04:40:00.794 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.59314324Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-04-26T04:40:00.794 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.594088807Z level=info msg="Executing migration" id="Drop category_id index" 2024-04-26T04:40:00.794 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.595822782Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-04-26T04:40:00.794 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.598479109Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-04-26T04:40:00.794 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.59899778Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-26T04:40:00.795 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.601772226Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-26T04:40:00.795 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.604741002Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-26T04:40:00.795 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.607535932Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-04-26T04:40:00.795 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.607978421Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-26T04:40:00.795 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.61056994Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-04-26T04:40:00.795 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.613371893Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-04-26T04:40:00.795 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.615958632Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-04-26T04:40:00.795 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.616289284Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-04-26T04:40:00.795 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.61913529Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-04-26T04:40:00.795 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.621874238Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-04-26T04:40:00.795 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.624454875Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-04-26T04:40:00.795 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.625060208Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-04-26T04:40:00.795 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.627751997Z level=info msg="Executing migration" id="Add epoch_end column" 2024-04-26T04:40:00.795 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.630497371Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-04-26T04:40:00.796 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.633061715Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-04-26T04:40:00.796 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.633408057Z level=info msg="Executing migration" id="Move region to single row" 2024-04-26T04:40:00.796 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.636029411Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-04-26T04:40:00.796 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.63647361Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-26T04:40:00.796 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.638768601Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-26T04:40:00.796 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.641397834Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-26T04:40:00.796 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.644085775Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-26T04:40:00.796 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.64680503Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-04-26T04:40:00.796 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.647419415Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-04-26T04:40:00.796 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.648320453Z level=info msg="Executing migration" id="create test_data table" 2024-04-26T04:40:00.796 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.648880776Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-04-26T04:40:00.796 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.649292227Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-04-26T04:40:00.797 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.650125249Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-26T04:40:00.797 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.651020378Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-04-26T04:40:00.797 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.651749422Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-04-26T04:40:00.797 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.652112383Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-04-26T04:40:00.797 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.652863385Z level=info msg="Executing migration" id="create team table" 2024-04-26T04:40:00.797 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.653288137Z level=info msg="Executing migration" id="add index team.org_id" 2024-04-26T04:40:00.797 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.653751873Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-04-26T04:40:00.797 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.654547029Z level=info msg="Executing migration" id="create team member table" 2024-04-26T04:40:00.797 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.654999946Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-04-26T04:40:00.797 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.655839679Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-04-26T04:40:00.797 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.656630085Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-04-26T04:40:00.798 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.657440796Z level=info msg="Executing migration" id="Add column email to team table" 2024-04-26T04:40:00.798 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.658202969Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-04-26T04:40:00.798 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.658994446Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-04-26T04:40:00.798 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.65986908Z level=info msg="Executing migration" id="create dashboard acl table" 2024-04-26T04:40:00.798 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.660891525Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-04-26T04:40:00.798 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.661550308Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-26T04:40:00.798 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.662432671Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-26T04:40:00.798 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.663362311Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-04-26T04:40:00.798 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.664072052Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-04-26T04:40:00.798 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.664532997Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-04-26T04:40:00.798 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.665480037Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-04-26T04:40:00.798 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.666297482Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-04-26T04:40:00.798 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.667250775Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-04-26T04:40:00.798 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.668001775Z level=info msg="Executing migration" id="create tag table" 2024-04-26T04:40:00.799 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.668834236Z level=info msg="Executing migration" id="add index tag.key_value" 2024-04-26T04:40:00.799 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.66961341Z level=info msg="Executing migration" id="create login attempt table" 2024-04-26T04:40:00.799 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.670496448Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-04-26T04:40:00.799 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.671490399Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-04-26T04:40:00.799 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.672168019Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-26T04:40:00.799 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.674084103Z level=info msg="Executing migration" id="create login_attempt v2" 2024-04-26T04:40:00.799 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.676925361Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-04-26T04:40:00.799 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.679775743Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-04-26T04:40:00.799 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.682424943Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-04-26T04:40:00.799 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.685117886Z level=info msg="Executing migration" id="create user auth table" 2024-04-26T04:40:00.799 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.687806325Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-26T04:40:00.799 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.690581205Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-04-26T04:40:00.799 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.693264215Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-04-26T04:40:00.799 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.696055291Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-04-26T04:40:00.800 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.698814185Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-04-26T04:40:00.800 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.701469123Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-04-26T04:40:00.800 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.70443484Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-04-26T04:40:00.800 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.707014838Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-04-26T04:40:00.800 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.70962386Z level=info msg="Executing migration" id="create server_lock table" 2024-04-26T04:40:00.800 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.71251638Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-04-26T04:40:00.800 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.715420178Z level=info msg="Executing migration" id="create user auth token table" 2024-04-26T04:40:00.800 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.718053831Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-04-26T04:40:00.800 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.72050623Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-04-26T04:40:00.800 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.721037778Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-04-26T04:40:00.801 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.721864249Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-04-26T04:40:00.801 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.722723515Z level=info msg="Executing migration" id="create cache_data table" 2024-04-26T04:40:00.801 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.723567339Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-04-26T04:40:00.801 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.724552655Z level=info msg="Executing migration" id="create short_url table v1" 2024-04-26T04:40:00.801 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.725333647Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-04-26T04:40:00.801 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.726112738Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-04-26T04:40:00.801 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.726457701Z level=info msg="Executing migration" id="delete alert_definition table" 2024-04-26T04:40:00.801 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.726747255Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-04-26T04:40:00.801 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.727238154Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-04-26T04:40:00.801 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.728069227Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-04-26T04:40:00.801 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.728580493Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-04-26T04:40:00.801 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.728893652Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-04-26T04:40:00.801 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.729348676Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-04-26T04:40:00.801 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.72976794Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-04-26T04:40:00.801 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.730254309Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-04-26T04:40:00.802 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.730718204Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-04-26T04:40:00.802 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.731550119Z level=info msg="Executing migration" id="drop alert_definition table" 2024-04-26T04:40:00.802 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.732045766Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-04-26T04:40:00.802 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.732540488Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-04-26T04:40:00.802 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.732970541Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-26T04:40:00.802 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.733470819Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-26T04:40:00.802 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.733917941Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-26T04:40:00.802 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.73421421Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-04-26T04:40:00.802 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.734625146Z level=info msg="Executing migration" id="create alert_instance table" 2024-04-26T04:40:00.802 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.735048134Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-26T04:40:00.802 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.735454795Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-26T04:40:00.802 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.735904898Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-04-26T04:40:00.802 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.736691648Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-26T04:40:00.803 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.737116675Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-04-26T04:40:00.803 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.737513174Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-04-26T04:40:00.803 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.740899613Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-04-26T04:40:00.803 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.744237591Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-26T04:40:00.803 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.744710759Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-04-26T04:40:00.803 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.745148738Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-04-26T04:40:00.803 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.745995126Z level=info msg="Executing migration" id="create alert_rule table" 2024-04-26T04:40:00.803 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.746417614Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-04-26T04:40:00.803 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.74687064Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-04-26T04:40:00.803 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.747307018Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-26T04:40:00.803 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.747755948Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-04-26T04:40:00.803 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.748063103Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-04-26T04:40:00.803 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.748919889Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-04-26T04:40:00.804 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.749723056Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-04-26T04:40:00.804 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.7505187Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-04-26T04:40:00.804 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.750981222Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-26T04:40:00.804 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.751431941Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-04-26T04:40:00.804 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.752249145Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-04-26T04:40:00.804 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.75312966Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-26T04:40:00.804 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.753595781Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-04-26T04:40:00.804 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.754412903Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-04-26T04:40:00.804 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.755254831Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-04-26T04:40:00.804 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.755585497Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-04-26T04:40:00.804 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.756072989Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-26T04:40:00.805 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.756861177Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-26T04:40:00.805 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.757340488Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-26T04:40:00.805 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.757627955Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-04-26T04:40:00.805 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.758506925Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-04-26T04:40:00.805 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.759374833Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-04-26T04:40:00.805 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.760262986Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-04-26T04:40:00.805 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.761142578Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-04-26T04:40:00.805 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.762027551Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-04-26T04:40:00.805 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.762887668Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-04-26T04:40:00.805 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.763913618Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-04-26T04:40:00.806 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.764779643Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-26T04:40:00.806 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.765686448Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-04-26T04:40:00.806 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.76844718Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-04-26T04:40:00.806 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.771237712Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-04-26T04:40:00.806 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.77400041Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-04-26T04:40:00.806 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.777118607Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-04-26T04:40:00.806 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.780007003Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-04-26T04:40:00.806 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.782562449Z level=info msg="Executing migration" id="create provenance_type table" 2024-04-26T04:40:00.806 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.785448333Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-04-26T04:40:00.806 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.788154688Z level=info msg="Executing migration" id="create alert_image table" 2024-04-26T04:40:00.806 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.790994366Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-04-26T04:40:00.807 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.793647126Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-04-26T04:40:01.050 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.796361324Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-04-26T04:40:01.051 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.799101171Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-04-26T04:40:01.051 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.799544966Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-04-26T04:40:01.051 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.799627102Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-04-26T04:40:01.051 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.802456459Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-04-26T04:40:01.051 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.80531099Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-04-26T04:40:01.051 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.807915373Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-04-26T04:40:01.051 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.808035619Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-04-26T04:40:01.051 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.808159249Z level=info msg="alerts found to migrate" alerts=0 2024-04-26T04:40:01.051 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.810799105Z level=info msg="Executing migration" id="create library_element table v1" 2024-04-26T04:40:01.051 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.813592076Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-04-26T04:40:01.051 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.816526234Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-04-26T04:40:01.051 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.818903891Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-04-26T04:40:01.051 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.821707721Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-04-26T04:40:01.052 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.824719436Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-04-26T04:40:01.052 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.827386774Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-04-26T04:40:01.052 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.829916843Z level=info msg="Executing migration" id="create data_keys table" 2024-04-26T04:40:01.052 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.832681849Z level=info msg="Executing migration" id="create secrets table" 2024-04-26T04:40:01.052 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.835328173Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-04-26T04:40:01.052 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.839636386Z level=info msg="Executing migration" id="add name column into data_keys" 2024-04-26T04:40:01.052 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.842459808Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-04-26T04:40:01.052 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.84515087Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-04-26T04:40:01.052 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.84946103Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-04-26T04:40:01.052 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.853702595Z level=info msg="Executing migration" id="create kv_store table v1" 2024-04-26T04:40:01.052 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.856520059Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-04-26T04:40:01.052 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.859115881Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-04-26T04:40:01.052 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.862001919Z level=info msg="Executing migration" id="create permission table" 2024-04-26T04:40:01.052 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.86473593Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-04-26T04:40:01.052 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.867469612Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-04-26T04:40:01.053 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.868119938Z level=info msg="Executing migration" id="create role table" 2024-04-26T04:40:01.053 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.869013432Z level=info msg="Executing migration" id="add column display_name" 2024-04-26T04:40:01.053 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.870004786Z level=info msg="Executing migration" id="add column group_name" 2024-04-26T04:40:01.053 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.871019288Z level=info msg="Executing migration" id="add index role.org_id" 2024-04-26T04:40:01.053 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.871988211Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-04-26T04:40:01.053 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.872706325Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-04-26T04:40:01.053 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.873439864Z level=info msg="Executing migration" id="create team role table" 2024-04-26T04:40:01.053 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.874348525Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-04-26T04:40:01.054 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.875143771Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-04-26T04:40:01.054 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.875921367Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-04-26T04:40:01.054 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.876744427Z level=info msg="Executing migration" id="create user role table" 2024-04-26T04:40:01.054 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.877724859Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-04-26T04:40:01.054 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.878486085Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-04-26T04:40:01.054 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.879318931Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-04-26T04:40:01.055 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.8800784Z level=info msg="Executing migration" id="create builtin role table" 2024-04-26T04:40:01.055 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.88092632Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-04-26T04:40:01.055 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.88194495Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-04-26T04:40:01.055 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.882651088Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-04-26T04:40:01.055 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.88363801Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-04-26T04:40:01.055 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.884148006Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-04-26T04:40:01.055 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.884622813Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-04-26T04:40:01.055 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.885075006Z level=info msg="Executing migration" id="add unique index role.uid" 2024-04-26T04:40:01.055 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.885502098Z level=info msg="Executing migration" id="create seed assignment table" 2024-04-26T04:40:01.055 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.885911426Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-04-26T04:40:01.055 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.886368822Z level=info msg="Executing migration" id="add column hidden to role table" 2024-04-26T04:40:01.055 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.887343256Z level=info msg="Executing migration" id="create query_history table v1" 2024-04-26T04:40:01.055 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.887759587Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-04-26T04:40:01.055 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.888214479Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-04-26T04:40:01.056 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.888498658Z level=info msg="Executing migration" id="teams permissions migration" 2024-04-26T04:40:01.056 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.888950145Z level=info msg="Executing migration" id="dashboard permissions" 2024-04-26T04:40:01.056 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.889441449Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-04-26T04:40:01.056 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.889829753Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-04-26T04:40:01.056 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.890144437Z level=info msg="Executing migration" id="alerting notification permissions" 2024-04-26T04:40:01.056 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.890483573Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-04-26T04:40:01.056 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.890900422Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-04-26T04:40:01.056 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.891369467Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-04-26T04:40:01.056 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.894317491Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-04-26T04:40:01.056 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.8968651Z level=info msg="Executing migration" id="create correlation table v1" 2024-04-26T04:40:01.056 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.89951691Z level=info msg="Executing migration" id="add index correlations.uid" 2024-04-26T04:40:01.056 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.90220678Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-04-26T04:40:01.056 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.904911467Z level=info msg="Executing migration" id="add correlation config column" 2024-04-26T04:40:01.056 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.9074259Z level=info msg="Executing migration" id="create entity_events table" 2024-04-26T04:40:01.057 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.910224182Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-04-26T04:40:01.057 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.912908158Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-26T04:40:01.057 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.913121926Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-26T04:40:01.057 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.915688714Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T04:40:01.057 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.915874767Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T04:40:01.057 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.918251258Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-04-26T04:40:01.057 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.921081338Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-04-26T04:40:01.057 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.923750487Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-04-26T04:40:01.057 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.926520525Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T04:40:01.057 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.929306011Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-04-26T04:40:01.057 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.931729081Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-26T04:40:01.057 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.93426044Z level=info msg="Executing migration" id="Drop public config table" 2024-04-26T04:40:01.057 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.936981568Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-04-26T04:40:01.058 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.93982975Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-04-26T04:40:01.058 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.942472413Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-26T04:40:01.058 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.945099659Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-04-26T04:40:01.058 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.947846624Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-04-26T04:40:01.058 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.951217474Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-04-26T04:40:01.058 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.952257897Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-04-26T04:40:01.058 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.953326573Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-04-26T04:40:01.058 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.954112644Z level=info msg="Executing migration" id="add share column" 2024-04-26T04:40:01.058 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.955145018Z level=info msg="Executing migration" id="create default alerting folders" 2024-04-26T04:40:01.058 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.956001359Z level=info msg="Executing migration" id="create file table" 2024-04-26T04:40:01.058 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.956816305Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-04-26T04:40:01.058 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.957540907Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-04-26T04:40:01.058 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.958365337Z level=info msg="Executing migration" id="create file_meta table" 2024-04-26T04:40:01.059 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.959284406Z level=info msg="Executing migration" id="file table idx: path key" 2024-04-26T04:40:01.059 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.960091439Z level=info msg="Executing migration" id="set path collation in file table" 2024-04-26T04:40:01.059 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.960744189Z level=info msg="Executing migration" id="managed permissions migration" 2024-04-26T04:40:01.059 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.961548581Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-04-26T04:40:01.059 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.962504243Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-04-26T04:40:01.059 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.963213883Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-04-26T04:40:01.059 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.964268277Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-04-26T04:40:01.059 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.96509495Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-04-26T04:40:01.059 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.965886473Z level=info msg="Executing migration" id="update group index for alert rules" 2024-04-26T04:40:01.059 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.966870855Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-04-26T04:40:01.059 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.967768422Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-04-26T04:40:01.059 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.968434322Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-04-26T04:40:01.059 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.969558879Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-04-26T04:40:01.059 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.970606627Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-04-26T04:40:01.060 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.971563974Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-04-26T04:40:01.060 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.981451655Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-04-26T04:40:01.060 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.982221385Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-04-26T04:40:01.060 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.983093897Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-04-26T04:40:01.060 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.986631667Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-04-26T04:40:01.060 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.987459046Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-04-26T04:40:01.060 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.988356521Z level=info msg="Executing migration" id="create folder table" 2024-04-26T04:40:01.060 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.989040476Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-04-26T04:40:01.060 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.989909548Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-04-26T04:40:01.060 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.990798423Z level=info msg="Executing migration" id="Update folder title length" 2024-04-26T04:40:01.060 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.991547213Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-04-26T04:40:01.060 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=migrator t=2024-04-26T04:40:00.992326135Z level=info msg="migrations completed" performed=485 skipped=0 duration=706.347743ms 2024-04-26T04:40:01.060 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=sqlstore t=2024-04-26T04:40:00.992771765Z level=info msg="Created default organization" 2024-04-26T04:40:01.061 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=plugin.loader t=2024-04-26T04:40:01.049832536Z level=info msg="Plugin registered" pluginID=input 2024-04-26T04:40:01.308 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=plugin.loader t=2024-04-26T04:40:01.078663372Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-04-26T04:40:01.309 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=plugin.loader t=2024-04-26T04:40:01.078715542Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-04-26T04:40:01.309 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=secrets t=2024-04-26T04:40:01.07875066Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-04-26T04:40:01.309 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=query_data t=2024-04-26T04:40:01.080450049Z level=info msg="Query Service initialization" 2024-04-26T04:40:01.309 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=live.push_http t=2024-04-26T04:40:01.082723028Z level=info msg="Live Push Gateway initialization" 2024-04-26T04:40:01.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:01 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:01.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:01 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:01.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:01 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:01.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:01 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:01.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:01 smithi156 ceph-mon[39130]: Deploying daemon node-exporter.a on smithi156 2024-04-26T04:40:01.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:01 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:01.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:01 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:01.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:01 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:01.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:01 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:01.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:01 smithi156 ceph-mon[27570]: Deploying daemon node-exporter.a on smithi156 2024-04-26T04:40:01.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:01.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:01.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:01.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:01.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-mon[31139]: Deploying daemon node-exporter.a on smithi156 2024-04-26T04:40:02.250 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=infra.usagestats.collector t=2024-04-26T04:40:01.818550869Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-04-26T04:40:02.250 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=provisioning.datasources t=2024-04-26T04:40:01.818916649Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-26T04:40:02.250 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=provisioning.datasources t=2024-04-26T04:40:01.852714696Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-26T04:40:02.250 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=provisioning.datasources t=2024-04-26T04:40:01.884772085Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-04-26T04:40:02.250 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=provisioning.alerting t=2024-04-26T04:40:01.906321127Z level=info msg="starting to provision alerting" 2024-04-26T04:40:02.250 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=provisioning.alerting t=2024-04-26T04:40:01.906339603Z level=info msg="finished to provision alerting" 2024-04-26T04:40:02.250 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=grafanaStorageLogger t=2024-04-26T04:40:01.906498196Z level=info msg="storage starting" 2024-04-26T04:40:02.250 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=ngalert.state.manager t=2024-04-26T04:40:01.906520856Z level=info msg="Warming state cache for startup" 2024-04-26T04:40:02.250 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=http.server t=2024-04-26T04:40:01.908268236Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-04-26T04:40:02.251 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=sqlstore.transactions t=2024-04-26T04:40:01.917378608Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-26T04:40:02.251 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=ngalert.state.manager t=2024-04-26T04:40:01.91760974Z level=info msg="State cache has been initialized" states=0 duration=11.088007ms 2024-04-26T04:40:02.251 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=ticker t=2024-04-26T04:40:01.917672751Z level=info msg=starting first_tick=2024-04-26T04:40:10Z 2024-04-26T04:40:02.251 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=ngalert.multiorg.alertmanager t=2024-04-26T04:40:01.917668853Z level=info msg="starting MultiOrg Alertmanager" 2024-04-26T04:40:02.251 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=sqlstore.transactions t=2024-04-26T04:40:01.958528408Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-26T04:40:02.251 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:40:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[62306]: logger=sqlstore.transactions t=2024-04-26T04:40:01.999426877Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-26T04:40:02.520 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:02 smithi156 ceph-mon[27570]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:02.520 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:02 smithi156 ceph-mon[39130]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:02.550 INFO:teuthology.orchestra.run.smithi156.stdout:34359738416 2024-04-26T04:40:02.550 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-26T04:40:02.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:02 smithi171 ceph-mon[31139]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:02.963 INFO:teuthology.orchestra.run.smithi156.stdout:163208757273 2024-04-26T04:40:02.963 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-26T04:40:03.517 INFO:teuthology.orchestra.run.smithi156.stdout:77309411367 2024-04-26T04:40:03.517 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-26T04:40:04.396 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:04 smithi171 ceph-mon[31139]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:04.396 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:04 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:04.566 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:04 smithi156 ceph-mon[27570]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:04.567 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:04 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:04.567 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:04 smithi156 ceph-mon[39130]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:04.567 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:04 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:04.595 INFO:teuthology.orchestra.run.smithi156.stdout:55834574893 2024-04-26T04:40:04.596 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-26T04:40:04.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:40:04 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:40:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:40:05.842 INFO:teuthology.orchestra.run.smithi156.stdout:128849018910 2024-04-26T04:40:05.842 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-26T04:40:06.343 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:06 smithi156 systemd[1]: Starting Ceph node-exporter.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:40:06.372 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:06.374 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:06.375 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:06.375 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:06.386 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:06.415 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:06.655 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:06 smithi156 ceph-mon[27570]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:06.656 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:06 smithi156 ceph-mon[39130]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:06.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:06 smithi171 ceph-mon[31139]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:08.359 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:08 smithi156 bash[73052]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-26T04:40:08.664 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:08 smithi156 ceph-mon[39130]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:08.664 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:08 smithi156 ceph-mon[27570]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:08.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:08 smithi171 ceph-mon[31139]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:08.959 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:08 smithi156 bash[73052]: Getting image source signatures 2024-04-26T04:40:08.959 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:08 smithi156 bash[73052]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-26T04:40:08.959 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:08 smithi156 bash[73052]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-26T04:40:08.959 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:08 smithi156 bash[73052]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-26T04:40:10.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:10 smithi156 ceph-mon[27570]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:10.594 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:10 smithi156 bash[73052]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-26T04:40:10.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:10 smithi156 ceph-mon[39130]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:10.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:10 smithi171 ceph-mon[31139]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:11.093 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:10 smithi156 bash[73052]: Writing manifest to image destination 2024-04-26T04:40:11.593 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:11 smithi156 podman[73052]: 2024-04-26 04:40:11.137300656 +0000 UTC m=+3.085705964 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:40:11.957 INFO:teuthology.orchestra.run.smithi156.stdout:231928234000 2024-04-26T04:40:11.957 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-26T04:40:12.489 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:12 smithi156 ceph-mon[39130]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:12.490 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:12 smithi156 ceph-mon[27570]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:12.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:12 smithi171 ceph-mon[31139]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:13.767 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:13.947 INFO:teuthology.orchestra.run.smithi156.stdout:77309411369 2024-04-26T04:40:14.594 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:14 smithi156 podman[73052]: 2024-04-26 04:40:14.258254328 +0000 UTC m=+6.206659632 container create 71daaa840a85f2b13d03559d7a2873cbd0a533b3d3f453c522cf55cc5ee67449 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:40:15.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:14 smithi156 ceph-mon[27570]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:15.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:14 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1219347298' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:40:15.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:14 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:15.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:14 smithi156 ceph-mon[39130]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:15.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:14 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1219347298' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:40:15.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:14 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:15.376 INFO:teuthology.orchestra.run.smithi156.stdout:163208757275 2024-04-26T04:40:15.377 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411367 got 77309411369 for osd.2 2024-04-26T04:40:15.377 DEBUG:teuthology.parallel:result is None 2024-04-26T04:40:15.378 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:40:14 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:40:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:40:15.378 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:14 smithi171 ceph-mon[31139]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:15.378 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:14 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1219347298' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:40:15.378 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:14 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:15.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/828264981' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:40:15.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/828264981' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:40:15.594 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 podman[73052]: 2024-04-26 04:40:15.351193315 +0000 UTC m=+7.299598630 container init 71daaa840a85f2b13d03559d7a2873cbd0a533b3d3f453c522cf55cc5ee67449 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:40:15.594 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 podman[73052]: 2024-04-26 04:40:15.357227037 +0000 UTC m=+7.305632342 container start 71daaa840a85f2b13d03559d7a2873cbd0a533b3d3f453c522cf55cc5ee67449 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:40:15.594 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.360Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:40:15.594 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.360Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:40:15.594 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.361Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:40:15.595 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.361Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:40:15.595 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.361Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:40:15.595 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.361Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:40:15.595 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:40:15.595 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:40:15.595 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:40:15.595 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:40:15.595 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:40:15.595 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:40:15.595 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:40:15.595 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:40:15.595 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:40:15.595 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:40:15.595 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:40:15.595 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:40:15.596 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:40:15.596 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:40:15.596 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:40:15.596 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:40:15.596 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:40:15.596 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:40:15.596 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:40:15.596 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:40:15.596 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:40:15.596 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:40:15.596 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:40:15.596 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:40:15.596 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:40:15.596 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:40:15.596 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:40:15.597 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:40:15.597 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:40:15.597 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:40:15.597 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:40:15.597 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:40:15.597 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:40:15.597 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:40:15.597 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:40:15.597 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:40:15.597 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:40:15.597 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:40:15.597 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:40:15.597 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:40:15.597 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:40:15.598 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:40:15.598 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:40:15.598 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:40:15.598 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:40:15.598 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:40:15.598 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[73516]: ts=2024-04-26T04:40:15.362Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:40:15.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:15 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/828264981' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:40:16.005 INFO:teuthology.orchestra.run.smithi156.stdout:34359738420 2024-04-26T04:40:16.304 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757273 got 163208757275 for osd.5 2024-04-26T04:40:16.305 DEBUG:teuthology.parallel:result is None 2024-04-26T04:40:16.558 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:16 smithi156 ceph-mon[27570]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:16.558 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:16 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3392255806' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:40:16.559 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:16 smithi156 ceph-mon[39130]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:16.559 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:16 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3392255806' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:40:16.559 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:16 smithi156 bash[73052]: 71daaa840a85f2b13d03559d7a2873cbd0a533b3d3f453c522cf55cc5ee67449 2024-04-26T04:40:16.559 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:40:16 smithi156 systemd[1]: Started Ceph node-exporter.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:40:16.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:16 smithi171 ceph-mon[31139]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:16.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:16 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3392255806' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:40:17.121 INFO:teuthology.orchestra.run.smithi156.stdout:197568495639 2024-04-26T04:40:17.439 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:17 smithi171 bash[62586]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-26T04:40:17.700 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:17 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:17.700 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:17 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:17.700 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:17 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:17.700 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:17 smithi156 ceph-mon[39130]: Deploying daemon node-exporter.b on smithi171 2024-04-26T04:40:17.700 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:17 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2363460458' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:40:17.700 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:17 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:17.700 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:17 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:17.700 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:17 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:17.700 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:17 smithi156 ceph-mon[27570]: Deploying daemon node-exporter.b on smithi171 2024-04-26T04:40:17.701 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:17 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2363460458' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:40:17.724 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495635 got 197568495639 for osd.6 2024-04-26T04:40:17.724 DEBUG:teuthology.parallel:result is None 2024-04-26T04:40:17.746 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:17 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:17.746 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:17 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:17.746 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:17 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:17.747 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:17 smithi171 ceph-mon[31139]: Deploying daemon node-exporter.b on smithi171 2024-04-26T04:40:17.747 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:17 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2363460458' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:40:17.999 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:17 smithi171 bash[62586]: Getting image source signatures 2024-04-26T04:40:17.999 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:17 smithi171 bash[62586]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-26T04:40:17.999 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:17 smithi171 bash[62586]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-26T04:40:18.000 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:17 smithi171 bash[62586]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-26T04:40:18.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:18 smithi156 ceph-mon[27570]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:18.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:18 smithi156 ceph-mon[39130]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:18.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:18 smithi171 ceph-mon[31139]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:19.695 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 bash[62586]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-26T04:40:19.695 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 bash[62586]: Writing manifest to image destination 2024-04-26T04:40:19.959 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 podman[62586]: 2024-04-26 04:40:19.624126244 +0000 UTC m=+2.352398312 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:40:19.960 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 podman[62586]: 2024-04-26 04:40:19.748299179 +0000 UTC m=+2.476571248 container create 5ef699e7d857d217f9bf8a73a55e118a6613daddd1f25d073a93edcb222c1657 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:40:19.960 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 podman[62586]: 2024-04-26 04:40:19.959383291 +0000 UTC m=+2.687655351 container init 5ef699e7d857d217f9bf8a73a55e118a6613daddd1f25d073a93edcb222c1657 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:40:20.250 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 podman[62586]: 2024-04-26 04:40:19.96509859 +0000 UTC m=+2.693370649 container start 5ef699e7d857d217f9bf8a73a55e118a6613daddd1f25d073a93edcb222c1657 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:40:20.250 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.969Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:40:20.250 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.969Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:40:20.250 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.969Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:40:20.250 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.969Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:40:20.250 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:40:20.250 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:40:20.250 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:40:20.250 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:40:20.251 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:40:20.251 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:40:20.251 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:40:20.251 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:40:20.251 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:40:20.251 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:40:20.251 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:40:20.251 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:40:20.252 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:40:20.252 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:40:20.252 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:40:20.252 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:40:20.252 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:40:20.252 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:40:20.252 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:40:20.252 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:40:20.252 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:40:20.252 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:40:20.252 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:40:20.252 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:40:20.252 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:40:20.253 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:40:20.253 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:40:20.253 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:40:20.253 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:40:20.253 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:40:20.253 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:40:20.253 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:40:20.253 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:40:20.253 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:40:20.253 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:40:20.253 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:40:20.253 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:40:20.253 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:40:20.253 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:40:20.253 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:40:20.254 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:40:20.254 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:40:20.254 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:40:20.254 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:40:20.254 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:40:20.254 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:40:20.254 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.970Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:40:20.254 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.971Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:40:20.254 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[62639]: ts=2024-04-26T04:40:19.971Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:40:20.254 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:20 smithi171 bash[62586]: 5ef699e7d857d217f9bf8a73a55e118a6613daddd1f25d073a93edcb222c1657 2024-04-26T04:40:20.254 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:40:20 smithi171 systemd[1]: Started Ceph node-exporter.b for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:40:20.584 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738420 for osd.0 2024-04-26T04:40:20.585 DEBUG:teuthology.parallel:result is None 2024-04-26T04:40:20.774 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:20 smithi156 ceph-mon[39130]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:20.774 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:20 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2949866549' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:40:20.775 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:20 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:20.775 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:20 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:20.775 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:20 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:20.775 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:20 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:20.775 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:20 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:20.775 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:20 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:20.775 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:20 smithi156 ceph-mon[27570]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:20.775 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:20 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2949866549' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:40:20.775 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:20 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:20.775 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:20 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:20.775 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:20 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:20.775 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:20 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:20.776 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:20 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:20.776 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:20 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:20.832 INFO:teuthology.orchestra.run.smithi156.stdout:55834574896 2024-04-26T04:40:20.904 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:20 smithi171 ceph-mon[31139]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:20.905 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:20 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2949866549' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:40:20.905 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:20 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:20.905 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:20 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:20.905 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:20 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:20.905 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:20 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:20.905 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:20 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:20.905 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:20 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:21.014 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:21.436 INFO:teuthology.orchestra.run.smithi156.stdout:128849018913 2024-04-26T04:40:21.646 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574893 got 55834574896 for osd.1 2024-04-26T04:40:21.646 DEBUG:teuthology.parallel:result is None 2024-04-26T04:40:21.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:21 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/542400105' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:40:21.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:21 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/542400105' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:40:21.858 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:21 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/542400105' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:40:22.763 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:22 smithi156 ceph-mon[39130]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:22.763 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:22 smithi156 ceph-mon[27570]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:22.787 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018910 got 128849018913 for osd.4 2024-04-26T04:40:22.787 DEBUG:teuthology.parallel:result is None 2024-04-26T04:40:22.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:22 smithi171 ceph-mon[31139]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:23.634 INFO:teuthology.orchestra.run.smithi156.stdout:107374182439 2024-04-26T04:40:24.557 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:24 smithi156 ceph-mon[27570]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:24.557 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:24 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/674564352' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:40:24.557 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:24 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:24.557 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:24 smithi156 ceph-mon[39130]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:24.557 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:24 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/674564352' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:40:24.557 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:24 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:24.582 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182434 got 107374182439 for osd.3 2024-04-26T04:40:24.582 DEBUG:teuthology.parallel:result is None 2024-04-26T04:40:24.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:24 smithi171 ceph-mon[31139]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:24.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:24 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/674564352' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:40:24.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:24 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:24.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:40:24 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:40:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:40:25.248 INFO:teuthology.orchestra.run.smithi156.stdout:231928234003 2024-04-26T04:40:25.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:25 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3632427798' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:40:25.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:25 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3632427798' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:40:25.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:25 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3632427798' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:40:26.287 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234000 got 231928234003 for osd.7 2024-04-26T04:40:26.287 DEBUG:teuthology.parallel:result is None 2024-04-26T04:40:26.287 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-26T04:40:26.288 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:40:26.586 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:26.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:26 smithi156 ceph-mon[39130]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:26.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:26 smithi156 ceph-mon[27570]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:26.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:26 smithi171 ceph-mon[31139]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:28.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:28 smithi156 ceph-mon[27570]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:28.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:28 smithi156 ceph-mon[39130]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:29.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:28 smithi171 ceph-mon[31139]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:29.282 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:40:29.283 INFO:teuthology.orchestra.run.smithi156.stderr:dumped all 2024-04-26T04:40:29.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:29 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:29.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:29 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:29.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:29 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:30.140 INFO:teuthology.orchestra.run.smithi156.stdout:{"pg_ready":true,"pg_map":{"version":36,"stamp":"2024-04-26T04:40:29.204623+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385524,"kb_used_data":6660,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575692,"statfs":{"total":767960285184,"available":765517508608,"internally_reserved":0,"allocated":6819840,"data_stored":3876964,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001768"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":12,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:31.235562+0000","last_change":"2024-04-26T04:39:29.193470+0000","last_active":"2024-04-26T04:39:31.235562+0000","last_peered":"2024-04-26T04:39:31.235562+0000","last_clean":"2024-04-26T04:39:31.235562+0000","last_became_active":"2024-04-26T04:39:29.193156+0000","last_became_peered":"2024-04-26T04:39:29.193156+0000","last_unstale":"2024-04-26T04:39:31.235562+0000","last_undegraded":"2024-04-26T04:39:31.235562+0000","last_fullsized":"2024-04-26T04:39:31.235562+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:16:39.309651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":12,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:31.202453+0000","last_change":"2024-04-26T04:39:29.193876+0000","last_active":"2024-04-26T04:39:31.202453+0000","last_peered":"2024-04-26T04:39:31.202453+0000","last_clean":"2024-04-26T04:39:31.202453+0000","last_became_active":"2024-04-26T04:39:29.193532+0000","last_became_peered":"2024-04-26T04:39:29.193532+0000","last_unstale":"2024-04-26T04:39:31.202453+0000","last_undegraded":"2024-04-26T04:39:31.202453+0000","last_fullsized":"2024-04-26T04:39:31.202453+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:58:26.728897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":12,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:31.202961+0000","last_change":"2024-04-26T04:39:29.224943+0000","last_active":"2024-04-26T04:39:31.202961+0000","last_peered":"2024-04-26T04:39:31.202961+0000","last_clean":"2024-04-26T04:39:31.202961+0000","last_became_active":"2024-04-26T04:39:29.224586+0000","last_became_peered":"2024-04-26T04:39:29.224586+0000","last_unstale":"2024-04-26T04:39:31.202961+0000","last_undegraded":"2024-04-26T04:39:31.202961+0000","last_fullsized":"2024-04-26T04:39:31.202961+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:50:41.653796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":12,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:31.202898+0000","last_change":"2024-04-26T04:39:29.224807+0000","last_active":"2024-04-26T04:39:31.202898+0000","last_peered":"2024-04-26T04:39:31.202898+0000","last_clean":"2024-04-26T04:39:31.202898+0000","last_became_active":"2024-04-26T04:39:29.224497+0000","last_became_peered":"2024-04-26T04:39:29.224497+0000","last_unstale":"2024-04-26T04:39:31.202898+0000","last_undegraded":"2024-04-26T04:39:31.202898+0000","last_fullsized":"2024-04-26T04:39:31.202898+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:20:03.848049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"62'128","reported_seq":270,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:31.261979+0000","last_change":"2024-04-26T04:39:00.628070+0000","last_active":"2024-04-26T04:39:31.261979+0000","last_peered":"2024-04-26T04:39:31.261979+0000","last_clean":"2024-04-26T04:39:31.261979+0000","last_became_active":"2024-04-26T04:39:00.627546+0000","last_became_peered":"2024-04-26T04:39:00.627546+0000","last_unstale":"2024-04-26T04:39:31.261979+0000","last_undegraded":"2024-04-26T04:39:31.261979+0000","last_fullsized":"2024-04-26T04:39:31.261979+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:36:55.213599+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:36:55.213599+0000","last_clean_scrub_stamp":"2024-04-26T04:36:55.213599+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:26:44.288000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":12,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:32.306769+0000","last_change":"2024-04-26T04:39:30.840097+0000","last_active":"2024-04-26T04:39:32.306769+0000","last_peered":"2024-04-26T04:39:32.306769+0000","last_clean":"2024-04-26T04:39:32.306769+0000","last_became_active":"2024-04-26T04:39:30.839750+0000","last_became_peered":"2024-04-26T04:39:30.839750+0000","last_unstale":"2024-04-26T04:39:32.306769+0000","last_undegraded":"2024-04-26T04:39:32.306769+0000","last_fullsized":"2024-04-26T04:39:32.306769+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:00:14.008690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":12,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:33.190473+0000","last_change":"2024-04-26T04:39:29.224560+0000","last_active":"2024-04-26T04:39:33.190473+0000","last_peered":"2024-04-26T04:39:33.190473+0000","last_clean":"2024-04-26T04:39:33.190473+0000","last_became_active":"2024-04-26T04:39:29.224239+0000","last_became_peered":"2024-04-26T04:39:29.224239+0000","last_unstale":"2024-04-26T04:39:33.190473+0000","last_undegraded":"2024-04-26T04:39:33.190473+0000","last_fullsized":"2024-04-26T04:39:33.190473+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:27:28.097185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":12,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:32.306769+0000","last_change":"2024-04-26T04:39:30.545245+0000","last_active":"2024-04-26T04:39:32.306769+0000","last_peered":"2024-04-26T04:39:32.306769+0000","last_clean":"2024-04-26T04:39:32.306769+0000","last_became_active":"2024-04-26T04:39:30.544940+0000","last_became_peered":"2024-04-26T04:39:30.544940+0000","last_unstale":"2024-04-26T04:39:32.306769+0000","last_undegraded":"2024-04-26T04:39:32.306769+0000","last_fullsized":"2024-04-26T04:39:32.306769+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:53:32.749890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":12,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:31.202400+0000","last_change":"2024-04-26T04:39:29.224546+0000","last_active":"2024-04-26T04:39:31.202400+0000","last_peered":"2024-04-26T04:39:31.202400+0000","last_clean":"2024-04-26T04:39:31.202400+0000","last_became_active":"2024-04-26T04:39:29.224238+0000","last_became_peered":"2024-04-26T04:39:29.224238+0000","last_unstale":"2024-04-26T04:39:31.202400+0000","last_undegraded":"2024-04-26T04:39:31.202400+0000","last_fullsized":"2024-04-26T04:39:31.202400+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:45:24.471564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":5,"up_from":38,"seq":163208757278,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":630784,"data_stored":263307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:07 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.51400000000000001,"15min":0.50800000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.75},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.495,"15min":0.48899999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.66600000000000004}]},{"osd":1,"last update":"Fri Apr 26 04:40:07 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.53700000000000003,"15min":0.53900000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.69899999999999995,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.51000000000000001,"15min":0.50700000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.70399999999999996}]},{"osd":2,"last update":"Fri Apr 26 04:40:07 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.51600000000000001,"15min":0.51400000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.747,"5min":0.88,"15min":0.88},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.54900000000000004,"15min":0.54500000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.71499999999999997}]},{"osd":3,"last update":"Fri Apr 26 04:40:07 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54500000000000004,"15min":0.54700000000000004},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.52600000000000002,"15min":0.52000000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.69499999999999995}]},{"osd":4,"last update":"Fri Apr 26 04:40:07 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.41499999999999998,"15min":0.40100000000000002},"min":{"1min":0.32000000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.103,"5min":1.103,"15min":1.103},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.42099999999999999,"15min":0.40699999999999997},"min":{"1min":0.313,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.0249999999999999,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.58099999999999996}]},{"osd":6,"last update":"Fri Apr 26 04:39:32 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.159,"5min":1.159,"15min":1.159},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":1.133,"5min":1.133,"15min":1.133},"last":0.59799999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:39:59 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.623}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.35499999999999998,"15min":0.33700000000000002},"min":{"1min":0.254,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.60199999999999998,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.36799999999999999,"15min":0.35399999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.64700000000000002,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.45300000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.39300000000000002,"15min":0.38400000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.376,"5min":0.38500000000000001,"15min":0.379},"min":{"1min":0.28999999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.503,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.32000000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:39:22 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.42799999999999999,"15min":0.42599999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.439,"15min":0.434},"min":{"1min":0.36099999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.35399999999999998}]},{"osd":4,"last update":"Fri Apr 26 04:39:46 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56799999999999995,"15min":0.56999999999999995},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.57799999999999996,"15min":0.57499999999999996},"min":{"1min":0.441,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.499}]},{"osd":5,"last update":"Fri Apr 26 04:40:09 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.626,"15min":0.63200000000000001},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.624,"15min":0.63},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.80300000000000005,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.46800000000000003}]},{"osd":6,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.48899999999999999}]},{"osd":7,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.51200000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411372,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":630784,"data_stored":263307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:01 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.36699999999999999},"min":{"1min":0.26000000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.436},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.40300000000000002,"15min":0.39600000000000002},"min":{"1min":0.27700000000000002,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.751,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.377}]},{"osd":1,"last update":"Fri Apr 26 04:40:01 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.39400000000000002,"15min":0.38400000000000001},"min":{"1min":0.26400000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.42199999999999999,"15min":0.41199999999999998},"min":{"1min":0.28499999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.42599999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:40:24 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.46700000000000003,"15min":0.47299999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.49299999999999999,"15min":0.505},"min":{"1min":0.34300000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.495}]},{"osd":4,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.56100000000000005,"15min":0.56799999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.751,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.59799999999999998,"15min":0.60199999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.66100000000000003}]},{"osd":5,"last update":"Fri Apr 26 04:40:12 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.57699999999999996,"15min":0.57299999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.54400000000000004,"15min":0.54600000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.61699999999999999}]},{"osd":6,"last update":"Fri Apr 26 04:39:36 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.72299999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:40:01 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.70499999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1224704,"data_stored":853675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:39:39 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.35299999999999998,"15min":0.33700000000000002},"min":{"1min":0.23499999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.53900000000000003,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.38200000000000001,"15min":0.35599999999999998},"min":{"1min":0.29399999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.61499999999999999,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.30099999999999999}]},{"osd":2,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.41799999999999998,"15min":0.40200000000000002},"min":{"1min":0.311,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.53900000000000003,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.40500000000000003,"15min":0.39500000000000002},"min":{"1min":0.28299999999999997,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.54600000000000004,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.39700000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:39:25 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.5,"15min":0.496},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.33100000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:39:44 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.57799999999999996,"15min":0.58499999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.69899999999999995,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.60099999999999998,"15min":0.60499999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.76100000000000001,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.46300000000000002}]},{"osd":5,"last update":"Fri Apr 26 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.58599999999999997,"15min":0.59599999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.64100000000000001,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.501},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.64300000000000002,"15min":0.65400000000000003},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.74199999999999999,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.497}]},{"osd":6,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.48099999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.58799999999999997}]}]},{"osd":7,"up_from":54,"seq":231928234004,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1220608,"data_stored":853352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.51400000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.502}]},{"osd":2,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.38700000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.39800000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.32100000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.34999999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.375}]}]},{"osd":3,"up_from":25,"seq":107374182440,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:24 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.38400000000000001,"15min":0.38500000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.219},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.40600000000000003,"15min":0.40799999999999997},"min":{"1min":0.29999999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.55800000000000005,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.314}]},{"osd":1,"last update":"Fri Apr 26 04:39:24 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.44900000000000001,"15min":0.45200000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.60799999999999998,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.251},{"interface":"front","average":{"1min":0.438,"5min":0.442,"15min":0.442},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.57399999999999995,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.33200000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:39:24 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.45500000000000002,"15min":0.45900000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.41799999999999998,"15min":0.42199999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.503,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.30299999999999999}]},{"osd":4,"last update":"Fri Apr 26 04:39:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.55900000000000005,"15min":0.56299999999999994},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.68799999999999994,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.55100000000000005,"15min":0.55600000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.66100000000000003,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.50700000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:40:07 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.56699999999999995,"15min":0.56999999999999995},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.56699999999999995,"15min":0.56599999999999995},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.432}]},{"osd":6,"last update":"Fri Apr 26 04:39:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.46300000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:39:56 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.498},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.441}]}]},{"osd":4,"up_from":30,"seq":128849018915,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":630784,"data_stored":263307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:48 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.51100000000000001,"15min":0.50600000000000001},"min":{"1min":0.38700000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.51400000000000001,"15min":0.51200000000000001},"min":{"1min":0.39100000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.64300000000000002,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.54100000000000004}]},{"osd":1,"last update":"Fri Apr 26 04:39:48 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.51000000000000001,"15min":0.51200000000000001},"min":{"1min":0.34999999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.51700000000000002,"15min":0.51300000000000001},"min":{"1min":0.38300000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.64900000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:39:48 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.53900000000000003,"15min":0.53700000000000003},"min":{"1min":0.38700000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.53700000000000003,"15min":0.53900000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.75,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.63}]},{"osd":3,"last update":"Fri Apr 26 04:39:48 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.54000000000000004,"15min":0.54400000000000004},"min":{"1min":0.38600000000000001,"5min":0.372,"15min":0.372},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.40699999999999997,"5min":0.377,"15min":0.377},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.51300000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:40:10 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.45500000000000002,"15min":0.45200000000000001},"min":{"1min":0.34200000000000003,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.64700000000000002,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.45700000000000002,"15min":0.44700000000000001},"min":{"1min":0.33600000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.497}]},{"osd":6,"last update":"Fri Apr 26 04:39:34 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.47899999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:39:56 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.45400000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495641,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1220608,"data_stored":853402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.46999999999999997}]},{"osd":1,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.48099999999999998}]},{"osd":2,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.45600000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.434}]},{"osd":4,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.44700000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.48999999999999999}]},{"osd":7,"last update":"Fri Apr 26 04:39:59 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.502},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.50900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:40:30.142 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:40:30.664 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[27570]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:30.664 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[27570]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:40:30.664 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:30.664 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:30.664 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:30.664 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:30.664 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:30.664 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:30.665 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:30.665 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[39130]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:30.665 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[39130]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:40:30.665 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:30.665 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:30.665 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:30.665 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:30.665 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:30.665 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:30.665 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:30 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:30.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:30 smithi171 ceph-mon[31139]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:31.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:30 smithi171 ceph-mon[31139]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:40:31.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:30 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:31.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:30 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:31.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:30 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:31.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:30 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:31.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:31.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:30 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:31.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:30 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:31.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:31 smithi156 ceph-mon[27570]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:40:31.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:31 smithi156 ceph-mon[27570]: Reconfiguring daemon alertmanager.a on smithi156 2024-04-26T04:40:31.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:31 smithi156 ceph-mon[39130]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:40:31.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:31 smithi156 ceph-mon[39130]: Reconfiguring daemon alertmanager.a on smithi156 2024-04-26T04:40:31.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:31 smithi171 ceph-mon[31139]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:40:31.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:31 smithi171 ceph-mon[31139]: Reconfiguring daemon alertmanager.a on smithi156 2024-04-26T04:40:32.493 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:32 smithi156 ceph-mon[27570]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:32.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:32 smithi156 ceph-mon[39130]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:32.956 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:32 smithi171 ceph-mon[31139]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:33.220 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:32 smithi156 systemd[1]: Stopping Ceph alertmanager.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:40:34.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:34 smithi171 ceph-mon[31139]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:34.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:40:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:40:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:40:34.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:34 smithi156 ceph-mon[27570]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:34.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:34 smithi156 ceph-mon[39130]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:34.919 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:40:34.921 INFO:teuthology.orchestra.run.smithi156.stderr:dumped all 2024-04-26T04:40:35.189 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[70425]: ts=2024-04-26T04:40:34.865Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:40:35.189 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:34 smithi156 podman[75413]: 2024-04-26 04:40:34.896342354 +0000 UTC m=+0.163920910 container died 0dea4b285c05366b14edace4597f7785d7083ffe0a03e53c1fc64e2073ccf8a6 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:40:35.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:35 smithi156 podman[75413]: 2024-04-26 04:40:35.412991611 +0000 UTC m=+0.680570157 container cleanup 0dea4b285c05366b14edace4597f7785d7083ffe0a03e53c1fc64e2073ccf8a6 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:40:35.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:35 smithi156 bash[75413]: ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T04:40:36.077 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:35 smithi156 podman[75432]: 2024-04-26 04:40:35.614518609 +0000 UTC m=+0.719740798 container remove 0dea4b285c05366b14edace4597f7785d7083ffe0a03e53c1fc64e2073ccf8a6 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:40:36.077 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:35 smithi156 podman[75432]: 2024-04-26 04:40:35.715332479 +0000 UTC m=+0.820554668 volume remove 2571df10f3e87aa4b42a444049676ca7e5e4d6c83f97eca9db8fdbd4bef4e987 2024-04-26T04:40:36.377 INFO:teuthology.orchestra.run.smithi156.stdout:{"pg_ready":true,"pg_map":{"version":38,"stamp":"2024-04-26T04:40:33.205443+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385524,"kb_used_data":6660,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575692,"statfs":{"total":767960285184,"available":765517508608,"internally_reserved":0,"allocated":6819840,"data_stored":3876964,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001972"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":12,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:31.235562+0000","last_change":"2024-04-26T04:39:29.193470+0000","last_active":"2024-04-26T04:39:31.235562+0000","last_peered":"2024-04-26T04:39:31.235562+0000","last_clean":"2024-04-26T04:39:31.235562+0000","last_became_active":"2024-04-26T04:39:29.193156+0000","last_became_peered":"2024-04-26T04:39:29.193156+0000","last_unstale":"2024-04-26T04:39:31.235562+0000","last_undegraded":"2024-04-26T04:39:31.235562+0000","last_fullsized":"2024-04-26T04:39:31.235562+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:16:39.309651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":12,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:31.202453+0000","last_change":"2024-04-26T04:39:29.193876+0000","last_active":"2024-04-26T04:39:31.202453+0000","last_peered":"2024-04-26T04:39:31.202453+0000","last_clean":"2024-04-26T04:39:31.202453+0000","last_became_active":"2024-04-26T04:39:29.193532+0000","last_became_peered":"2024-04-26T04:39:29.193532+0000","last_unstale":"2024-04-26T04:39:31.202453+0000","last_undegraded":"2024-04-26T04:39:31.202453+0000","last_fullsized":"2024-04-26T04:39:31.202453+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:58:26.728897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":12,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:31.202961+0000","last_change":"2024-04-26T04:39:29.224943+0000","last_active":"2024-04-26T04:39:31.202961+0000","last_peered":"2024-04-26T04:39:31.202961+0000","last_clean":"2024-04-26T04:39:31.202961+0000","last_became_active":"2024-04-26T04:39:29.224586+0000","last_became_peered":"2024-04-26T04:39:29.224586+0000","last_unstale":"2024-04-26T04:39:31.202961+0000","last_undegraded":"2024-04-26T04:39:31.202961+0000","last_fullsized":"2024-04-26T04:39:31.202961+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:50:41.653796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":12,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:31.202898+0000","last_change":"2024-04-26T04:39:29.224807+0000","last_active":"2024-04-26T04:39:31.202898+0000","last_peered":"2024-04-26T04:39:31.202898+0000","last_clean":"2024-04-26T04:39:31.202898+0000","last_became_active":"2024-04-26T04:39:29.224497+0000","last_became_peered":"2024-04-26T04:39:29.224497+0000","last_unstale":"2024-04-26T04:39:31.202898+0000","last_undegraded":"2024-04-26T04:39:31.202898+0000","last_fullsized":"2024-04-26T04:39:31.202898+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:20:03.848049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"62'128","reported_seq":270,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:31.261979+0000","last_change":"2024-04-26T04:39:00.628070+0000","last_active":"2024-04-26T04:39:31.261979+0000","last_peered":"2024-04-26T04:39:31.261979+0000","last_clean":"2024-04-26T04:39:31.261979+0000","last_became_active":"2024-04-26T04:39:00.627546+0000","last_became_peered":"2024-04-26T04:39:00.627546+0000","last_unstale":"2024-04-26T04:39:31.261979+0000","last_undegraded":"2024-04-26T04:39:31.261979+0000","last_fullsized":"2024-04-26T04:39:31.261979+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:36:55.213599+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:36:55.213599+0000","last_clean_scrub_stamp":"2024-04-26T04:36:55.213599+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:26:44.288000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":12,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:32.306769+0000","last_change":"2024-04-26T04:39:30.840097+0000","last_active":"2024-04-26T04:39:32.306769+0000","last_peered":"2024-04-26T04:39:32.306769+0000","last_clean":"2024-04-26T04:39:32.306769+0000","last_became_active":"2024-04-26T04:39:30.839750+0000","last_became_peered":"2024-04-26T04:39:30.839750+0000","last_unstale":"2024-04-26T04:39:32.306769+0000","last_undegraded":"2024-04-26T04:39:32.306769+0000","last_fullsized":"2024-04-26T04:39:32.306769+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:00:14.008690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":12,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:33.190473+0000","last_change":"2024-04-26T04:39:29.224560+0000","last_active":"2024-04-26T04:39:33.190473+0000","last_peered":"2024-04-26T04:39:33.190473+0000","last_clean":"2024-04-26T04:39:33.190473+0000","last_became_active":"2024-04-26T04:39:29.224239+0000","last_became_peered":"2024-04-26T04:39:29.224239+0000","last_unstale":"2024-04-26T04:39:33.190473+0000","last_undegraded":"2024-04-26T04:39:33.190473+0000","last_fullsized":"2024-04-26T04:39:33.190473+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:27:28.097185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":12,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:32.306769+0000","last_change":"2024-04-26T04:39:30.545245+0000","last_active":"2024-04-26T04:39:32.306769+0000","last_peered":"2024-04-26T04:39:32.306769+0000","last_clean":"2024-04-26T04:39:32.306769+0000","last_became_active":"2024-04-26T04:39:30.544940+0000","last_became_peered":"2024-04-26T04:39:30.544940+0000","last_unstale":"2024-04-26T04:39:32.306769+0000","last_undegraded":"2024-04-26T04:39:32.306769+0000","last_fullsized":"2024-04-26T04:39:32.306769+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:53:32.749890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":12,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-26T04:39:31.202400+0000","last_change":"2024-04-26T04:39:29.224546+0000","last_active":"2024-04-26T04:39:31.202400+0000","last_peered":"2024-04-26T04:39:31.202400+0000","last_clean":"2024-04-26T04:39:31.202400+0000","last_became_active":"2024-04-26T04:39:29.224238+0000","last_became_peered":"2024-04-26T04:39:29.224238+0000","last_unstale":"2024-04-26T04:39:31.202400+0000","last_undegraded":"2024-04-26T04:39:31.202400+0000","last_fullsized":"2024-04-26T04:39:31.202400+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:45:24.471564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":5,"up_from":38,"seq":163208757279,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":630784,"data_stored":263307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:07 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.51400000000000001,"15min":0.50800000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.495,"15min":0.48899999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.58799999999999997}]},{"osd":1,"last update":"Fri Apr 26 04:40:07 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.53700000000000003,"15min":0.53900000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.69899999999999995,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.375},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.51000000000000001,"15min":0.50700000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.46999999999999997}]},{"osd":2,"last update":"Fri Apr 26 04:40:07 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.51600000000000001,"15min":0.51400000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.747,"5min":0.88,"15min":0.88},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.54900000000000004,"15min":0.54500000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.45800000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:40:07 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54500000000000004,"15min":0.54700000000000004},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.437},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.52600000000000002,"15min":0.52000000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.50700000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:40:07 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.41499999999999998,"15min":0.40100000000000002},"min":{"1min":0.32000000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.103,"5min":1.103,"15min":1.103},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.504,"5min":0.42099999999999999,"15min":0.40699999999999997},"min":{"1min":0.313,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.0249999999999999,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.53800000000000003}]},{"osd":6,"last update":"Fri Apr 26 04:39:32 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.159,"5min":1.159,"15min":1.159},"last":0.627},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":1.133,"5min":1.133,"15min":1.133},"last":0.60699999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:39:59 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.54900000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.35499999999999998,"15min":0.33700000000000002},"min":{"1min":0.254,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.60199999999999998,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.36799999999999999,"15min":0.35399999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.64700000000000002,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.42999999999999999}]},{"osd":2,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.39300000000000002,"15min":0.38400000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.376,"5min":0.38500000000000001,"15min":0.379},"min":{"1min":0.28999999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.503,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.46000000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:40:26 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.41799999999999998,"15min":0.42299999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.53000000000000003,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.441},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.58299999999999996,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.41599999999999998}]},{"osd":4,"last update":"Fri Apr 26 04:39:46 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56799999999999995,"15min":0.56999999999999995},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.57799999999999996,"15min":0.57499999999999996},"min":{"1min":0.441,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.56299999999999994}]},{"osd":5,"last update":"Fri Apr 26 04:40:09 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.626,"15min":0.63200000000000001},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.624,"15min":0.63},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.80300000000000005,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.65300000000000002}]},{"osd":6,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.63100000000000001}]},{"osd":7,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.69599999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":630784,"data_stored":263307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:01 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.36699999999999999},"min":{"1min":0.26000000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.37},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.40300000000000002,"15min":0.39600000000000002},"min":{"1min":0.27700000000000002,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.751,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.27200000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:40:01 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.39400000000000002,"15min":0.38400000000000001},"min":{"1min":0.26400000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.42199999999999999,"15min":0.41199999999999998},"min":{"1min":0.28499999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.36099999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:40:24 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.46700000000000003,"15min":0.47299999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.49299999999999999,"15min":0.505},"min":{"1min":0.34300000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.33500000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.56100000000000005,"15min":0.56799999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.751,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.59799999999999998,"15min":0.60199999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.55000000000000004}]},{"osd":5,"last update":"Fri Apr 26 04:40:12 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.57699999999999996,"15min":0.57299999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.54400000000000004,"15min":0.54600000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.622}]},{"osd":6,"last update":"Fri Apr 26 04:39:36 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.58999999999999997}]},{"osd":7,"last update":"Fri Apr 26 04:40:01 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.57099999999999995}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1224704,"data_stored":853675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:39:39 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.35299999999999998,"15min":0.33700000000000002},"min":{"1min":0.23499999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.53900000000000003,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.38200000000000001,"15min":0.35599999999999998},"min":{"1min":0.29399999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.61499999999999999,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.45900000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.41799999999999998,"15min":0.40200000000000002},"min":{"1min":0.311,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.53900000000000003,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.40500000000000003,"15min":0.39500000000000002},"min":{"1min":0.28299999999999997,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.54600000000000004,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.44500000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:40:26 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.48499999999999999,"15min":0.49099999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.57499999999999996,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.44900000000000001,"15min":0.45700000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.54800000000000004,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.371}]},{"osd":4,"last update":"Fri Apr 26 04:39:44 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.57799999999999996,"15min":0.58499999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.69899999999999995,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.60099999999999998,"15min":0.60499999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.76100000000000001,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.54800000000000004}]},{"osd":5,"last update":"Fri Apr 26 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.58599999999999997,"15min":0.59599999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.64100000000000001,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.64300000000000002,"15min":0.65400000000000003},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.74199999999999999,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.59599999999999997}]},{"osd":6,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.625}]},{"osd":7,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.502},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.56799999999999995}]}]},{"osd":7,"up_from":54,"seq":231928234005,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1220608,"data_stored":853352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.51400000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.502}]},{"osd":2,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.38700000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.39800000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.32100000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.34999999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.375}]}]},{"osd":3,"up_from":25,"seq":107374182441,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:26 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.38200000000000001,"15min":0.38400000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.248},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.40200000000000002,"15min":0.40600000000000003},"min":{"1min":0.29099999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.60899999999999999,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.32200000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:40:26 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.439,"15min":0.44900000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.432,"15min":0.439},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.308}]},{"osd":2,"last update":"Fri Apr 26 04:40:26 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.44700000000000001,"15min":0.45700000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.41399999999999998,"15min":0.41999999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.54600000000000004,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.28599999999999998}]},{"osd":4,"last update":"Fri Apr 26 04:39:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.55900000000000005,"15min":0.56299999999999994},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.68799999999999994,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.55100000000000005,"15min":0.55600000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.66100000000000003,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.55600000000000005}]},{"osd":5,"last update":"Fri Apr 26 04:40:07 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.56699999999999995,"15min":0.56999999999999995},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.56699999999999995,"15min":0.56599999999999995},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.59699999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:39:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.56699999999999995}]},{"osd":7,"last update":"Fri Apr 26 04:39:56 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.54000000000000004}]}]},{"osd":4,"up_from":30,"seq":128849018916,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":630784,"data_stored":263307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:48 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.51100000000000001,"15min":0.50600000000000001},"min":{"1min":0.38700000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.51400000000000001,"15min":0.51200000000000001},"min":{"1min":0.39100000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.64300000000000002,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.54100000000000004}]},{"osd":1,"last update":"Fri Apr 26 04:39:48 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.51000000000000001,"15min":0.51200000000000001},"min":{"1min":0.34999999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.51700000000000002,"15min":0.51300000000000001},"min":{"1min":0.38300000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.64900000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:39:48 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.53900000000000003,"15min":0.53700000000000003},"min":{"1min":0.38700000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.53700000000000003,"15min":0.53900000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.75,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.63}]},{"osd":3,"last update":"Fri Apr 26 04:39:48 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.54000000000000004,"15min":0.54400000000000004},"min":{"1min":0.38600000000000001,"5min":0.372,"15min":0.372},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.40699999999999997,"5min":0.377,"15min":0.377},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.51300000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:40:10 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.45500000000000002,"15min":0.45200000000000001},"min":{"1min":0.34200000000000003,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.64700000000000002,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.45700000000000002,"15min":0.44700000000000001},"min":{"1min":0.33600000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.497}]},{"osd":6,"last update":"Fri Apr 26 04:39:34 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.47899999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:39:56 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.45400000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495642,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1220608,"data_stored":853402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.41699999999999998}]},{"osd":1,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.433},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.59499999999999997}]},{"osd":2,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.66800000000000004}]},{"osd":3,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.48499999999999999}]},{"osd":4,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.65100000000000002}]},{"osd":5,"last update":"Fri Apr 26 04:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.63},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.72499999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:39:59 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.69799999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:40:36.379 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-26T04:40:36.379 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-26T04:40:36.379 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-26T04:40:36.379 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:40:36.709 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:36 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T04:40:36.709 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:36 smithi156 systemd[1]: Stopped Ceph alertmanager.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:40:36.709 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:36 smithi156 systemd[1]: Starting Ceph alertmanager.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:40:36.710 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:36 smithi156 ceph-mon[27570]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:40:36.710 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:36 smithi156 ceph-mon[27570]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:36.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:36 smithi156 ceph-mon[39130]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:40:36.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:36 smithi156 ceph-mon[39130]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:36.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:36 smithi171 ceph-mon[31139]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:40:36.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:36 smithi171 ceph-mon[31139]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:37.022 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:36 smithi156 podman[75571]: 2024-04-26 04:40:36.611375168 +0000 UTC m=+0.023433865 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-26T04:40:37.023 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:36 smithi156 podman[75571]: 2024-04-26 04:40:36.723764327 +0000 UTC m=+0.135823026 volume create 15e029a4af625eede17cf0e28f932443c408a6ccbe722d2a6b8f74bcd08210c0 2024-04-26T04:40:37.023 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:36 smithi156 podman[75571]: 2024-04-26 04:40:36.828566334 +0000 UTC m=+0.240625072 container create 9742e32fd5fc5725890b5f85bf5498c37aff42d5c948228490cfb7a06b29a091 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:40:37.306 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:37.430 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:37 smithi156 podman[75571]: 2024-04-26 04:40:37.098519962 +0000 UTC m=+0.510578659 container init 9742e32fd5fc5725890b5f85bf5498c37aff42d5c948228490cfb7a06b29a091 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:40:37.430 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:37 smithi156 podman[75571]: 2024-04-26 04:40:37.104138523 +0000 UTC m=+0.516197220 container start 9742e32fd5fc5725890b5f85bf5498c37aff42d5c948228490cfb7a06b29a091 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:40:37.430 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:37 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[75609]: ts=2024-04-26T04:40:37.116Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-26T04:40:37.430 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:37 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[75609]: ts=2024-04-26T04:40:37.116Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-26T04:40:37.430 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:37 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[75609]: ts=2024-04-26T04:40:37.117Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.156 port=9094 2024-04-26T04:40:37.430 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:37 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[75609]: ts=2024-04-26T04:40:37.118Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T04:40:37.430 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:37 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[75609]: ts=2024-04-26T04:40:37.140Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:40:37.430 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:37 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[75609]: ts=2024-04-26T04:40:37.141Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:40:37.430 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:37 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[75609]: ts=2024-04-26T04:40:37.143Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-26T04:40:37.430 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:37 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[75609]: ts=2024-04-26T04:40:37.143Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-26T04:40:37.431 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:37 smithi156 bash[75571]: 9742e32fd5fc5725890b5f85bf5498c37aff42d5c948228490cfb7a06b29a091 2024-04-26T04:40:37.431 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:37 smithi156 systemd[1]: Started Ceph alertmanager.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:40:38.463 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:38 smithi156 ceph-mon[27570]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:38.463 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:38 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:38.464 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:38 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:38.464 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:38 smithi156 ceph-mon[27570]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:40:38.464 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:38 smithi156 ceph-mon[27570]: Reconfiguring daemon prometheus.a on smithi171 2024-04-26T04:40:38.464 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:38 smithi156 ceph-mon[39130]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:38.464 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:38 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:38.464 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:38 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:38.464 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:38 smithi156 ceph-mon[39130]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:40:38.464 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:38 smithi156 ceph-mon[39130]: Reconfiguring daemon prometheus.a on smithi171 2024-04-26T04:40:38.496 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:38 smithi171 ceph-mon[31139]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:38.496 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:38 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:38.497 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:38 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:38.497 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:38 smithi171 ceph-mon[31139]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:40:38.497 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:38 smithi171 ceph-mon[31139]: Reconfiguring daemon prometheus.a on smithi171 2024-04-26T04:40:39.267 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:40:39.267 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-26T04:40:39.551 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:39 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[75609]: ts=2024-04-26T04:40:39.118Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000532653s 2024-04-26T04:40:39.954 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-26T04:40:39.955 INFO:tasks.cephadm:Setup complete, yielding 2024-04-26T04:40:39.955 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-26T04:40:39.967 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi156.front.sepia.ceph.com 2024-04-26T04:40:39.967 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-04-26T04:40:40.227 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:40.408 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:40 smithi171 ceph-mon[31139]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:40.408 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:40 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1971073085' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:40:40.503 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:40 smithi156 ceph-mon[27570]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:40.503 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:40 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1971073085' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:40:40.504 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:40 smithi156 ceph-mon[39130]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:40.504 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:40 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1971073085' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:40:40.749 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:40 smithi171 systemd[1]: Stopping Ceph prometheus.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:40:41.130 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:40:40.856Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:40:41.131 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:40:40.856Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T04:40:41.131 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:40:40.856Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T04:40:41.131 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:40:40.856Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T04:40:41.131 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:40:40.856Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T04:40:41.131 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:40:40.856Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T04:40:41.131 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:40:40.856Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T04:40:41.131 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:40:40.856Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T04:40:41.131 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:40:40.856Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T04:40:41.131 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:40:40.856Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T04:40:41.131 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:40:40.856Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T04:40:41.131 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[60141]: ts=2024-04-26T04:40:40.856Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T04:40:41.131 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:40 smithi171 podman[63528]: 2024-04-26 04:40:40.867003129 +0000 UTC m=+0.152506794 container died 276c3aacfbc140f4427776fd3863a10db03f85eb4e3ae92650b07a5bb0741e8e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:40:41.480 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:41 smithi171 podman[63528]: 2024-04-26 04:40:41.220330641 +0000 UTC m=+0.505834295 container cleanup 276c3aacfbc140f4427776fd3863a10db03f85eb4e3ae92650b07a5bb0741e8e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:40:41.480 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:41 smithi171 bash[63528]: ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T04:40:41.750 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:41 smithi171 podman[63547]: 2024-04-26 04:40:41.480430418 +0000 UTC m=+0.613730669 container remove 276c3aacfbc140f4427776fd3863a10db03f85eb4e3ae92650b07a5bb0741e8e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:40:41.750 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:41 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T04:40:41.750 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:41 smithi171 systemd[1]: Stopped Ceph prometheus.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:40:41.750 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:41 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.084s CPU time. 2024-04-26T04:40:41.750 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:41 smithi171 systemd[1]: Starting Ceph prometheus.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:40:42.110 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:41 smithi171 podman[63647]: 2024-04-26 04:40:41.84838216 +0000 UTC m=+0.109902530 container create be422f8a7019247d962c70f402adf51675e40b2fa041bffa95920a63989de9c1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:40:42.110 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:41 smithi171 podman[63647]: 2024-04-26 04:40:41.758099536 +0000 UTC m=+0.019619922 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T04:40:42.110 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 podman[63647]: 2024-04-26 04:40:42.059111888 +0000 UTC m=+0.320632258 container init be422f8a7019247d962c70f402adf51675e40b2fa041bffa95920a63989de9c1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:40:42.110 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 podman[63647]: 2024-04-26 04:40:42.064295051 +0000 UTC m=+0.325815424 container start be422f8a7019247d962c70f402adf51675e40b2fa041bffa95920a63989de9c1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:40:42.111 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.088Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T04:40:42.111 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.088Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T04:40:42.111 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.088Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi171 (none))" 2024-04-26T04:40:42.111 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.088Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T04:40:42.111 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.088Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T04:40:42.111 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.090Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T04:40:42.111 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.090Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T04:40:42.111 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.091Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T04:40:42.111 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.091Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T04:40:42.111 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.092Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T04:40:42.111 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.092Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.377µs 2024-04-26T04:40:42.112 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.092Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T04:40:42.112 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.094Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-26T04:40:42.112 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.094Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-26T04:40:42.112 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.094Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=16.806µs wal_replay_duration=1.850706ms wbl_replay_duration=131ns total_replay_duration=1.885352ms 2024-04-26T04:40:42.112 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.095Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T04:40:42.112 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.095Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T04:40:42.112 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.095Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T04:40:42.112 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.109Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=13.995568ms db_storage=669ns remote_storage=1.032µs web_handler=343ns query_engine=562ns scrape=388.771µs scrape_sd=22.797µs notify=18.433µs notify_sd=5.695µs rules=13.353005ms tracing=5.348µs 2024-04-26T04:40:42.112 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.109Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T04:40:42.112 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:40:42.109Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T04:40:42.499 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 bash[63647]: be422f8a7019247d962c70f402adf51675e40b2fa041bffa95920a63989de9c1 2024-04-26T04:40:42.499 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:40:42 smithi171 systemd[1]: Started Ceph prometheus.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:40:42.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-mon[31139]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:42.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:42.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:42.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:40:42.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156:9093"}]: dispatch 2024-04-26T04:40:42.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:42.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:40:42.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:40:42.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:42.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:42 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:40:42.567 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[27570]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:42.567 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:42.567 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:42.567 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:40:42.567 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156:9093"}]: dispatch 2024-04-26T04:40:42.568 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:42.568 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:40:42.568 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:40:42.568 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:42.568 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:40:42.568 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[39130]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:42.568 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:42.568 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:42.568 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:40:42.568 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156:9093"}]: dispatch 2024-04-26T04:40:42.568 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:42.569 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:40:42.569 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:40:42.569 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:42.569 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:40:42.569 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE Bus STOPPING 2024-04-26T04:40:42.569 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:40:42.569 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE Bus STOPPED 2024-04-26T04:40:42.569 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE Bus STARTING 2024-04-26T04:40:42.569 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE Serving on http://:::9283 2024-04-26T04:40:42.569 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE Bus STARTED 2024-04-26T04:40:42.570 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE Bus STOPPING 2024-04-26T04:40:42.570 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:40:42.570 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE Bus STOPPED 2024-04-26T04:40:42.570 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE Bus STARTING 2024-04-26T04:40:42.570 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE Serving on http://:::9283 2024-04-26T04:40:42.570 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE Bus STARTED 2024-04-26T04:40:42.570 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE Bus STOPPING 2024-04-26T04:40:42.570 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:40:42.570 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE Bus STOPPED 2024-04-26T04:40:42.570 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE Bus STARTING 2024-04-26T04:40:42.570 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE Serving on http://:::9283 2024-04-26T04:40:42.570 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:40:42] ENGINE Bus STARTED 2024-04-26T04:40:43.240 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[27570]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:40:43.240 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[27570]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156:9093"}]: dispatch 2024-04-26T04:40:43.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:43 smithi171 ceph-mon[31139]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:40:43.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:43 smithi171 ceph-mon[31139]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156:9093"}]: dispatch 2024-04-26T04:40:43.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:43 smithi171 ceph-mon[31139]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:40:43.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:43 smithi171 ceph-mon[31139]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:40:43.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:43 smithi171 ceph-mon[31139]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:40:43.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:43 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:40:43.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:43 smithi171 ceph-mon[31139]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:40:43.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:43 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:43.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:43 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:43.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:43 smithi171 ceph-mon[31139]: from='client.14625 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:40:43.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:43 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:40:43.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:43 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:40:43.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[27570]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:40:43.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[27570]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:40:43.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[27570]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:40:43.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:40:43.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[27570]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:40:43.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:43.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:43.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[27570]: from='client.14625 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:40:43.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:40:43.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:40:43.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[39130]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:40:43.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[39130]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156:9093"}]: dispatch 2024-04-26T04:40:43.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[39130]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:40:43.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[39130]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:40:43.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[39130]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:40:43.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:40:43.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[39130]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:40:43.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:43.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:43.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[39130]: from='client.14625 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:40:43.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:40:43.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:43 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:40:44.316 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:44 smithi156 ceph-mon[39130]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:44.316 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:44 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-26T04:40:44.316 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:44 smithi156 ceph-mon[39130]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T04:40:44.316 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:44 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:40:44.316 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:44 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:40:44.317 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:44 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:44.317 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:44 smithi156 ceph-mon[27570]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:44.317 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:44 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-26T04:40:44.317 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:44 smithi156 ceph-mon[27570]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T04:40:44.317 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:44 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:40:44.317 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:44 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:40:44.317 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:44 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:44.317 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a[27547]: 2024-04-26T04:40:44.278+0000 7f8b40cb4700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:40:44.622 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:44 smithi171 ceph-mon[31139]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:44.622 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:44 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-26T04:40:44.622 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:44 smithi171 ceph-mon[31139]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T04:40:44.622 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:44 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:40:44.622 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:44 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:40:44.622 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:44 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:45.112 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-26T04:40:45.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-26T04:40:45.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[27570]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T04:40:45.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:40:45.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:40:45.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[27570]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:40:45.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[27570]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-26T04:40:45.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[27570]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T04:40:45.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-26T04:40:45.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[27570]: fsmap foo:0 2024-04-26T04:40:45.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[27570]: Saving service mds.foo spec with placement count:2 2024-04-26T04:40:45.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:45.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-26T04:40:45.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[39130]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T04:40:45.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:40:45.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:40:45.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[39130]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:40:45.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[39130]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-26T04:40:45.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[39130]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T04:40:45.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-26T04:40:45.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[39130]: fsmap foo:0 2024-04-26T04:40:45.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[39130]: Saving service mds.foo spec with placement count:2 2024-04-26T04:40:45.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:45 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:45.528 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:45 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-26T04:40:45.528 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:45 smithi171 ceph-mon[31139]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T04:40:45.528 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:45 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:40:45.528 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:45 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:40:45.528 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:45 smithi171 ceph-mon[31139]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:40:45.529 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:45 smithi171 ceph-mon[31139]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-26T04:40:45.529 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:45 smithi171 ceph-mon[31139]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T04:40:45.529 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:45 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-26T04:40:45.529 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:45 smithi171 ceph-mon[31139]: fsmap foo:0 2024-04-26T04:40:45.529 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:45 smithi171 ceph-mon[31139]: Saving service mds.foo spec with placement count:2 2024-04-26T04:40:45.529 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:45 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:45.978 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:46.384 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:46 smithi156 ceph-mon[27570]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:46.384 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:46 smithi156 ceph-mon[27570]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T04:40:46.384 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:46 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:46.384 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:46 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:46.384 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:46 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:46.385 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:46 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:46.385 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:46 smithi156 ceph-mon[39130]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:46.385 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:46 smithi156 ceph-mon[39130]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T04:40:46.385 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:46 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:46.385 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:46 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:46.386 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:46 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:46.386 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:46 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:46.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:46 smithi171 ceph-mon[31139]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:46.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:46 smithi171 ceph-mon[31139]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T04:40:46.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:46 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:46.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:46 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:46.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:46 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:46.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:46 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:47.284 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:40:47 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[75609]: ts=2024-04-26T04:40:47.121Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002809963s 2024-04-26T04:40:47.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:47 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:47.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:47 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:47.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:47 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:47.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:47 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.wyfzii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:47.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:47 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.wyfzii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:47.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:47 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.wyfzii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:40:47.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:47 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:47.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:47 smithi156 ceph-mon[27570]: Deploying daemon mds.foo.smithi156.wyfzii on smithi156 2024-04-26T04:40:47.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:47 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:47.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:47 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:47.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:47 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:47.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:47 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.wyfzii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:47.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:47 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.wyfzii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:47.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:47 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.wyfzii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:40:47.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:47 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:47.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:47 smithi156 ceph-mon[39130]: Deploying daemon mds.foo.smithi156.wyfzii on smithi156 2024-04-26T04:40:47.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:47 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:47.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:47 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:47.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:47 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:47.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:47 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.wyfzii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:47.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:47 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.wyfzii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:47.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:47 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.wyfzii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:40:47.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:47 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:47.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:47 smithi171 ceph-mon[31139]: Deploying daemon mds.foo.smithi156.wyfzii on smithi156 2024-04-26T04:40:48.676 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:48 smithi156 ceph-mon[27570]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:48.677 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:48 smithi156 ceph-mon[39130]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:48.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:48 smithi171 ceph-mon[31139]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:49.719 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-26T04:40:50.272 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:50.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:50 smithi156 ceph-mon[39130]: pgmap v50: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:50.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:50 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:50.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:50 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:50.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:50 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:50.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:50 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.kzjlqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:50.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:50 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.kzjlqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:50.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:50 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.kzjlqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:40:50.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:50 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:50.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:50 smithi156 ceph-mon[27570]: pgmap v50: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:50.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:50 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:50.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:50 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:50.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:50 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:50.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:50 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.kzjlqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:50.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:50 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.kzjlqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:50.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:50 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.kzjlqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:40:50.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:50 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:50.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:50 smithi171 ceph-mon[31139]: pgmap v50: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:50.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:50 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:50.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:50 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:50.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:50 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:50.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:50 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.kzjlqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:50.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:50 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.kzjlqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:50.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:50 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.kzjlqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:40:50.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:50 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:51.724 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[27570]: Deploying daemon mds.foo.smithi171.kzjlqn on smithi171 2024-04-26T04:40:51.724 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[27570]: mds.? [v2:172.21.15.156:6834/572371201,v1:172.21.15.156:6835/572371201] up:boot 2024-04-26T04:40:51.724 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[27570]: daemon mds.foo.smithi156.wyfzii assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-26T04:40:51.724 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[27570]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T04:40:51.724 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[27570]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-26T04:40:51.724 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[27570]: Cluster is now healthy 2024-04-26T04:40:51.724 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.wyfzii"}]: dispatch 2024-04-26T04:40:51.725 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[27570]: fsmap foo:0 1 up:standby 2024-04-26T04:40:51.725 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[27570]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:creating} 2024-04-26T04:40:51.725 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[27570]: daemon mds.foo.smithi156.wyfzii is now active in filesystem foo as rank 0 2024-04-26T04:40:51.725 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[39130]: Deploying daemon mds.foo.smithi171.kzjlqn on smithi171 2024-04-26T04:40:51.725 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[39130]: mds.? [v2:172.21.15.156:6834/572371201,v1:172.21.15.156:6835/572371201] up:boot 2024-04-26T04:40:51.725 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[39130]: daemon mds.foo.smithi156.wyfzii assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-26T04:40:51.725 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[39130]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T04:40:51.725 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[39130]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-26T04:40:51.725 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[39130]: Cluster is now healthy 2024-04-26T04:40:51.725 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.wyfzii"}]: dispatch 2024-04-26T04:40:51.725 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[39130]: fsmap foo:0 1 up:standby 2024-04-26T04:40:51.725 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[39130]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:creating} 2024-04-26T04:40:51.726 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:51 smithi156 ceph-mon[39130]: daemon mds.foo.smithi156.wyfzii is now active in filesystem foo as rank 0 2024-04-26T04:40:51.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:51 smithi171 ceph-mon[31139]: Deploying daemon mds.foo.smithi171.kzjlqn on smithi171 2024-04-26T04:40:51.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:51 smithi171 ceph-mon[31139]: mds.? [v2:172.21.15.156:6834/572371201,v1:172.21.15.156:6835/572371201] up:boot 2024-04-26T04:40:51.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:51 smithi171 ceph-mon[31139]: daemon mds.foo.smithi156.wyfzii assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-26T04:40:51.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:51 smithi171 ceph-mon[31139]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T04:40:51.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:51 smithi171 ceph-mon[31139]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-26T04:40:51.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:51 smithi171 ceph-mon[31139]: Cluster is now healthy 2024-04-26T04:40:51.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:51 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.wyfzii"}]: dispatch 2024-04-26T04:40:51.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:51 smithi171 ceph-mon[31139]: fsmap foo:0 1 up:standby 2024-04-26T04:40:51.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:51 smithi171 ceph-mon[31139]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:creating} 2024-04-26T04:40:51.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:51 smithi171 ceph-mon[31139]: daemon mds.foo.smithi156.wyfzii is now active in filesystem foo as rank 0 2024-04-26T04:40:52.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:52 smithi156 ceph-mon[27570]: pgmap v51: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:52.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:52 smithi156 ceph-mon[27570]: mds.? [v2:172.21.15.156:6834/572371201,v1:172.21.15.156:6835/572371201] up:active 2024-04-26T04:40:52.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:52 smithi156 ceph-mon[27570]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 2024-04-26T04:40:52.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:52 smithi156 ceph-mon[39130]: pgmap v51: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:52.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:52 smithi156 ceph-mon[39130]: mds.? [v2:172.21.15.156:6834/572371201,v1:172.21.15.156:6835/572371201] up:active 2024-04-26T04:40:52.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:52 smithi156 ceph-mon[39130]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 2024-04-26T04:40:52.675 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:52 smithi171 ceph-mon[31139]: pgmap v51: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:52.675 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:52 smithi171 ceph-mon[31139]: mds.? [v2:172.21.15.156:6834/572371201,v1:172.21.15.156:6835/572371201] up:active 2024-04-26T04:40:52.675 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:52 smithi171 ceph-mon[31139]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 2024-04-26T04:40:53.132 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-26T04:40:53.145 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-26T04:40:53.146 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-26T04:40:53.146 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-26T04:40:53.825 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:53.825 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:53.825 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:53.825 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:53.825 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:53.825 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:53.825 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[27570]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:53.825 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[27570]: mds.? [v2:172.21.15.171:6832/1083563586,v1:172.21.15.171:6833/1083563586] up:boot 2024-04-26T04:40:53.825 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.kzjlqn"}]: dispatch 2024-04-26T04:40:53.826 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[27570]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:40:53.826 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[27570]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:40:53.826 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:53.826 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:53.826 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:53.826 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:53.826 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:53.826 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:53.826 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[39130]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:53.826 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[39130]: mds.? [v2:172.21.15.171:6832/1083563586,v1:172.21.15.171:6833/1083563586] up:boot 2024-04-26T04:40:53.826 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.kzjlqn"}]: dispatch 2024-04-26T04:40:53.827 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[39130]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:40:53.827 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:53 smithi156 ceph-mon[39130]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:40:53.965 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:54.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:53 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:54.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:53 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:54.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:53 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:54.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:53 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:54.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:53 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:54.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:53 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:54.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:53 smithi171 ceph-mon[31139]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:54.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:53 smithi171 ceph-mon[31139]: mds.? [v2:172.21.15.171:6832/1083563586,v1:172.21.15.171:6833/1083563586] up:boot 2024-04-26T04:40:54.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:53 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.kzjlqn"}]: dispatch 2024-04-26T04:40:54.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:53 smithi171 ceph-mon[31139]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:40:54.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:53 smithi171 ceph-mon[31139]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:40:54.652 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:40:54 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:40:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:40:55.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:55 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:55.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:55 smithi156 ceph-mon[39130]: mds.? [v2:172.21.15.156:6834/572371201,v1:172.21.15.156:6835/572371201] up:active 2024-04-26T04:40:55.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:55 smithi156 ceph-mon[39130]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:40:55.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:55 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:55.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:55 smithi156 ceph-mon[27570]: mds.? [v2:172.21.15.156:6834/572371201,v1:172.21.15.156:6835/572371201] up:active 2024-04-26T04:40:55.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:55 smithi156 ceph-mon[27570]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:40:55.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:55 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:55.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:55 smithi171 ceph-mon[31139]: mds.? [v2:172.21.15.156:6834/572371201,v1:172.21.15.156:6835/572371201] up:active 2024-04-26T04:40:55.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:55 smithi171 ceph-mon[31139]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:40:56.127 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:40:56.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:56 smithi171 ceph-mon[31139]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-26T04:40:56.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:56 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2050139532' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:40:56.537 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:56 smithi156 ceph-mon[39130]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-26T04:40:56.537 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:56 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2050139532' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:40:56.537 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:56 smithi156 ceph-mon[27570]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-26T04:40:56.537 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:56 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2050139532' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:40:56.538 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:40:56 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:40:56] "GET /metrics HTTP/1.1" 200 205554 "" "Prometheus/2.43.0" 2024-04-26T04:40:57.354 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":20,"flags":0,"active_gid":24344,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6800","nonce":367046062},{"type":"v1","addr":"172.21.15.156:6801","nonce":367046062}]},"active_addr":"172.21.15.156:6801/367046062","active_change":"2024-04-26T04:39:29.182748+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24331,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.156:8443/","prometheus":"http://172.21.15.156:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":3291954384}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":32257782}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":1361591211}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":1772897570}]}]} 2024-04-26T04:40:57.359 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-26T04:40:57.359 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-26T04:40:57.359 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:40:57.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:57 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:57.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:57 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:57.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:57 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:57.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:57 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:57.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:57 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:57.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:57 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:58.178 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:58.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:58 smithi156 ceph-mon[27570]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:58.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:58 smithi156 ceph-mon[27570]: mds.? [v2:172.21.15.171:6832/1083563586,v1:172.21.15.171:6833/1083563586] up:standby 2024-04-26T04:40:58.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:58 smithi156 ceph-mon[27570]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:40:58.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:58 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:58.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:58 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:58.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:58 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:58.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:58 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:58.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:58 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:58.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:58 smithi156 ceph-mon[39130]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:58.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:58 smithi156 ceph-mon[39130]: mds.? [v2:172.21.15.171:6832/1083563586,v1:172.21.15.171:6833/1083563586] up:standby 2024-04-26T04:40:58.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:58 smithi156 ceph-mon[39130]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:40:58.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:58 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:58.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:58 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:58.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:58 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:58.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:58 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:58.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:58 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:58.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:58 smithi171 ceph-mon[31139]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:58.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:58 smithi171 ceph-mon[31139]: mds.? [v2:172.21.15.171:6832/1083563586,v1:172.21.15.171:6833/1083563586] up:standby 2024-04-26T04:40:59.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:58 smithi171 ceph-mon[31139]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:40:59.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:58 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:59.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:58 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:59.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:58 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:59.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:58 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:59.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:58 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:40:59.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:59 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:59.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:40:59 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:59.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:59 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:59.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:40:59 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:59.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:59 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:00.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:40:59 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:00.063 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:41:00.063 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":67,"fsid":"110027f2-0386-11ef-bc93-c7b262605968","created":"2024-04-26T04:33:46.638460+0000","modified":"2024-04-26T04:40:45.281111+0000","last_up_change":"2024-04-26T04:38:56.608997+0000","last_in_change":"2024-04-26T04:38:39.401044+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:36:55.213219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:39:28.969228+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:40:42.534967+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:40:43.505179+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"6b59fe49-0273-4115-91d9-6faea7d37616","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6803","nonce":3436276177}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6805","nonce":3436276177}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6809","nonce":3436276177}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6807","nonce":3436276177}]},"public_addr":"172.21.15.156:6803/3436276177","cluster_addr":"172.21.15.156:6805/3436276177","heartbeat_back_addr":"172.21.15.156:6809/3436276177","heartbeat_front_addr":"172.21.15.156:6807/3436276177","state":["exists","up"]},{"osd":1,"uuid":"c9673007-264f-4d7c-8234-bfd514e70d2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6811","nonce":273749030}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6813","nonce":273749030}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6817","nonce":273749030}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6815","nonce":273749030}]},"public_addr":"172.21.15.156:6811/273749030","cluster_addr":"172.21.15.156:6813/273749030","heartbeat_back_addr":"172.21.15.156:6817/273749030","heartbeat_front_addr":"172.21.15.156:6815/273749030","state":["exists","up"]},{"osd":2,"uuid":"d0051bbe-c0cd-42f4-9d8c-6b0a3e71886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6819","nonce":3653838089}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6821","nonce":3653838089}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6825","nonce":3653838089}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6823","nonce":3653838089}]},"public_addr":"172.21.15.156:6819/3653838089","cluster_addr":"172.21.15.156:6821/3653838089","heartbeat_back_addr":"172.21.15.156:6825/3653838089","heartbeat_front_addr":"172.21.15.156:6823/3653838089","state":["exists","up"]},{"osd":3,"uuid":"479e109c-1e4a-4124-9442-f85d63df6d04","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6827","nonce":2175305841}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6829","nonce":2175305841}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6832","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6833","nonce":2175305841}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6831","nonce":2175305841}]},"public_addr":"172.21.15.156:6827/2175305841","cluster_addr":"172.21.15.156:6829/2175305841","heartbeat_back_addr":"172.21.15.156:6833/2175305841","heartbeat_front_addr":"172.21.15.156:6831/2175305841","state":["exists","up"]},{"osd":4,"uuid":"b090dbe4-ed23-4015-a864-074aa64cace4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6801","nonce":2255415442}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6803","nonce":2255415442}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6807","nonce":2255415442}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6805","nonce":2255415442}]},"public_addr":"172.21.15.171:6801/2255415442","cluster_addr":"172.21.15.171:6803/2255415442","heartbeat_back_addr":"172.21.15.171:6807/2255415442","heartbeat_front_addr":"172.21.15.171:6805/2255415442","state":["exists","up"]},{"osd":5,"uuid":"f506481f-428b-40d6-a54b-8ef411672b53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6809","nonce":2373798416}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6811","nonce":2373798416}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6815","nonce":2373798416}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6813","nonce":2373798416}]},"public_addr":"172.21.15.171:6809/2373798416","cluster_addr":"172.21.15.171:6811/2373798416","heartbeat_back_addr":"172.21.15.171:6815/2373798416","heartbeat_front_addr":"172.21.15.171:6813/2373798416","state":["exists","up"]},{"osd":6,"uuid":"f1fd019a-372d-4b4e-bede-7659038df10a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6817","nonce":2328644410}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6819","nonce":2328644410}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6823","nonce":2328644410}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6821","nonce":2328644410}]},"public_addr":"172.21.15.171:6817/2328644410","cluster_addr":"172.21.15.171:6819/2328644410","heartbeat_back_addr":"172.21.15.171:6823/2328644410","heartbeat_front_addr":"172.21.15.171:6821/2328644410","state":["exists","up"]},{"osd":7,"uuid":"bee1398e-7912-458a-9f8b-279b84a22566","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6825","nonce":2507226151}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6827","nonce":2507226151}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6831","nonce":2507226151}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6829","nonce":2507226151}]},"public_addr":"172.21.15.171:6825/2507226151","cluster_addr":"172.21.15.171:6827/2507226151","heartbeat_back_addr":"172.21.15.171:6831/2507226151","heartbeat_front_addr":"172.21.15.171:6829/2507226151","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:03.400774+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:28.117542+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:52.563066+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:17.387470+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:40.460850+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:05.110287+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:29.602919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:55.143996+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:0/3755168891":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/3331144037":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/711432262":"2024-04-27T04:39:29.182492+0000","172.21.15.156:6801/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:6800/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/67599134":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/4143435073":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6800/1084840701":"2024-04-27T04:39:29.182492+0000","172.21.15.156:6801/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/402721227":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/2398977550":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6800/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6801/1084840701":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/314056408":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/1144243691":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/4047047180":"2024-04-27T04:34:42.908359+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:41:00.824 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:00 smithi156 ceph-mon[27570]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:41:00.824 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:00 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3997763159' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:41:00.824 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:00 smithi156 ceph-mon[39130]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:41:00.824 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:00 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3997763159' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:41:00.839 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:00 smithi171 ceph-mon[31139]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:41:00.840 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:00 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3997763159' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:41:01.222 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-26T04:41:01.222 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:41:01.508 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:02.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:02 smithi156 ceph-mon[27570]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:41:02.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:02 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:02.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:02 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:02.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:02 smithi156 ceph-mon[39130]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:41:02.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:02 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:02.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:02 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:02.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:02 smithi171 ceph-mon[31139]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:41:03.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:02 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:03.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:02 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:04.242 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:41:04.242 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":67,"fsid":"110027f2-0386-11ef-bc93-c7b262605968","created":"2024-04-26T04:33:46.638460+0000","modified":"2024-04-26T04:40:45.281111+0000","last_up_change":"2024-04-26T04:38:56.608997+0000","last_in_change":"2024-04-26T04:38:39.401044+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:36:55.213219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:39:28.969228+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:40:42.534967+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:40:43.505179+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"6b59fe49-0273-4115-91d9-6faea7d37616","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6803","nonce":3436276177}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6805","nonce":3436276177}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6809","nonce":3436276177}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":3436276177},{"type":"v1","addr":"172.21.15.156:6807","nonce":3436276177}]},"public_addr":"172.21.15.156:6803/3436276177","cluster_addr":"172.21.15.156:6805/3436276177","heartbeat_back_addr":"172.21.15.156:6809/3436276177","heartbeat_front_addr":"172.21.15.156:6807/3436276177","state":["exists","up"]},{"osd":1,"uuid":"c9673007-264f-4d7c-8234-bfd514e70d2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6811","nonce":273749030}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6813","nonce":273749030}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6817","nonce":273749030}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":273749030},{"type":"v1","addr":"172.21.15.156:6815","nonce":273749030}]},"public_addr":"172.21.15.156:6811/273749030","cluster_addr":"172.21.15.156:6813/273749030","heartbeat_back_addr":"172.21.15.156:6817/273749030","heartbeat_front_addr":"172.21.15.156:6815/273749030","state":["exists","up"]},{"osd":2,"uuid":"d0051bbe-c0cd-42f4-9d8c-6b0a3e71886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6819","nonce":3653838089}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6821","nonce":3653838089}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6825","nonce":3653838089}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":3653838089},{"type":"v1","addr":"172.21.15.156:6823","nonce":3653838089}]},"public_addr":"172.21.15.156:6819/3653838089","cluster_addr":"172.21.15.156:6821/3653838089","heartbeat_back_addr":"172.21.15.156:6825/3653838089","heartbeat_front_addr":"172.21.15.156:6823/3653838089","state":["exists","up"]},{"osd":3,"uuid":"479e109c-1e4a-4124-9442-f85d63df6d04","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6827","nonce":2175305841}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6829","nonce":2175305841}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6832","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6833","nonce":2175305841}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2175305841},{"type":"v1","addr":"172.21.15.156:6831","nonce":2175305841}]},"public_addr":"172.21.15.156:6827/2175305841","cluster_addr":"172.21.15.156:6829/2175305841","heartbeat_back_addr":"172.21.15.156:6833/2175305841","heartbeat_front_addr":"172.21.15.156:6831/2175305841","state":["exists","up"]},{"osd":4,"uuid":"b090dbe4-ed23-4015-a864-074aa64cace4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6801","nonce":2255415442}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6803","nonce":2255415442}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6807","nonce":2255415442}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6805","nonce":2255415442}]},"public_addr":"172.21.15.171:6801/2255415442","cluster_addr":"172.21.15.171:6803/2255415442","heartbeat_back_addr":"172.21.15.171:6807/2255415442","heartbeat_front_addr":"172.21.15.171:6805/2255415442","state":["exists","up"]},{"osd":5,"uuid":"f506481f-428b-40d6-a54b-8ef411672b53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6809","nonce":2373798416}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6811","nonce":2373798416}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6815","nonce":2373798416}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6813","nonce":2373798416}]},"public_addr":"172.21.15.171:6809/2373798416","cluster_addr":"172.21.15.171:6811/2373798416","heartbeat_back_addr":"172.21.15.171:6815/2373798416","heartbeat_front_addr":"172.21.15.171:6813/2373798416","state":["exists","up"]},{"osd":6,"uuid":"f1fd019a-372d-4b4e-bede-7659038df10a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6817","nonce":2328644410}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6819","nonce":2328644410}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6823","nonce":2328644410}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6821","nonce":2328644410}]},"public_addr":"172.21.15.171:6817/2328644410","cluster_addr":"172.21.15.171:6819/2328644410","heartbeat_back_addr":"172.21.15.171:6823/2328644410","heartbeat_front_addr":"172.21.15.171:6821/2328644410","state":["exists","up"]},{"osd":7,"uuid":"bee1398e-7912-458a-9f8b-279b84a22566","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6825","nonce":2507226151}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6827","nonce":2507226151}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6831","nonce":2507226151}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6829","nonce":2507226151}]},"public_addr":"172.21.15.171:6825/2507226151","cluster_addr":"172.21.15.171:6827/2507226151","heartbeat_back_addr":"172.21.15.171:6831/2507226151","heartbeat_front_addr":"172.21.15.171:6829/2507226151","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:03.400774+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:28.117542+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:52.563066+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:17.387470+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:40.460850+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:05.110287+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:29.602919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:55.143996+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:0/3755168891":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/3331144037":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/711432262":"2024-04-27T04:39:29.182492+0000","172.21.15.156:6801/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:6800/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/67599134":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/4143435073":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6800/1084840701":"2024-04-27T04:39:29.182492+0000","172.21.15.156:6801/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/402721227":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/2398977550":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6800/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6801/1084840701":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/314056408":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/1144243691":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/4047047180":"2024-04-27T04:34:42.908359+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:41:04.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:41:04 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:41:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:41:04.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:04 smithi171 ceph-mon[31139]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:41:04.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:04 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2754348291' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:41:04.831 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:04 smithi156 ceph-mon[27570]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:41:04.831 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:04 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2754348291' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:41:04.831 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:04 smithi156 ceph-mon[39130]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:41:04.831 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:04 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2754348291' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:41:05.224 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-26T04:41:05.224 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-26T04:41:05.225 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-26T04:41:05.225 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-26T04:41:05.225 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-26T04:41:05.226 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-26T04:41:05.226 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-26T04:41:05.226 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-26T04:41:05.789 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:05.790 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:05.790 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:05.790 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:05.791 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:05.793 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:05.796 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:05.800 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:06.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:05 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:06.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:05 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:06.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:05 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:06.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:05 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:06.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:05 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:06.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:05 smithi171 ceph-mon[31139]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:41:06.025 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:05 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:06.025 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:05 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:06.025 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:05 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:06.025 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:05 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:06.025 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:05 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:06.025 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:05 smithi156 ceph-mon[39130]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:41:06.025 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:05 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:06.025 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:05 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:06.025 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:05 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:06.026 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:05 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:06.026 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:05 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:06.026 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:05 smithi156 ceph-mon[27570]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:41:06.617 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:41:06 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:41:06] "GET /metrics HTTP/1.1" 200 239126 "" "Prometheus/2.43.0" 2024-04-26T04:41:08.508 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:08 smithi156 ceph-mon[27570]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:08.509 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:08 smithi156 ceph-mon[39130]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:08.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:08 smithi171 ceph-mon[31139]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:10.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:10 smithi156 ceph-mon[39130]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:10.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:10 smithi156 ceph-mon[27570]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:10.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:10 smithi171 ceph-mon[31139]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:12.531 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:12 smithi156 ceph-mon[39130]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:12.531 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:12 smithi156 ceph-mon[27570]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:12.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:12 smithi171 ceph-mon[31139]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:14.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:14 smithi156 ceph-mon[27570]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:14.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:14 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:14.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:14 smithi156 ceph-mon[39130]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:14.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:14 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:14.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:14 smithi171 ceph-mon[31139]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:14.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:14 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:14.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:41:14 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:41:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:41:16.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:16 smithi156 ceph-mon[27570]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:16.594 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:41:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:41:16] "GET /metrics HTTP/1.1" 200 239089 "" "Prometheus/2.43.0" 2024-04-26T04:41:16.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:16 smithi156 ceph-mon[39130]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:16.713 INFO:teuthology.orchestra.run.smithi156.stdout:55834574909 2024-04-26T04:41:16.713 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-26T04:41:16.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:16 smithi171 ceph-mon[31139]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:17.301 INFO:teuthology.orchestra.run.smithi156.stdout:197568495651 2024-04-26T04:41:17.301 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-26T04:41:18.511 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:18 smithi156 ceph-mon[27570]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:18.512 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:18 smithi156 ceph-mon[39130]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:18.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:18 smithi171 ceph-mon[31139]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:20.391 INFO:teuthology.orchestra.run.smithi156.stdout:107374182451 2024-04-26T04:41:20.391 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-26T04:41:20.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:20 smithi156 ceph-mon[27570]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:20.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:20 smithi156 ceph-mon[39130]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:20.739 INFO:teuthology.orchestra.run.smithi156.stdout:77309411382 2024-04-26T04:41:20.740 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-26T04:41:20.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:20 smithi171 ceph-mon[31139]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:21.456 INFO:teuthology.orchestra.run.smithi156.stdout:231928234015 2024-04-26T04:41:21.457 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-26T04:41:22.168 INFO:teuthology.orchestra.run.smithi156.stdout:163208757289 2024-04-26T04:41:22.168 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-26T04:41:22.469 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:22 smithi156 ceph-mon[27570]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:22.469 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:22 smithi156 ceph-mon[39130]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:22.552 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:22 smithi171 ceph-mon[31139]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:23.022 INFO:teuthology.orchestra.run.smithi156.stdout:34359738433 2024-04-26T04:41:23.022 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-26T04:41:23.294 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:23.320 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:23.321 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:23.321 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:23.321 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:23.329 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:24.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:24 smithi156 ceph-mon[27570]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:24.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:24 smithi156 ceph-mon[39130]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:24.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:24 smithi171 ceph-mon[31139]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:24.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:41:24 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:41:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:41:24.912 INFO:teuthology.orchestra.run.smithi156.stdout:128849018927 2024-04-26T04:41:24.913 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-26T04:41:26.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:26 smithi156 ceph-mon[27570]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:26.593 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:41:26 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:41:26] "GET /metrics HTTP/1.1" 200 239089 "" "Prometheus/2.43.0" 2024-04-26T04:41:26.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:26 smithi156 ceph-mon[39130]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:26.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:26 smithi171 ceph-mon[31139]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:28.447 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:28 smithi156 ceph-mon[39130]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:28.447 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:28 smithi156 ceph-mon[27570]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:28.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:28 smithi171 ceph-mon[31139]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:29.186 INFO:teuthology.orchestra.run.smithi156.stdout:163208757291 2024-04-26T04:41:29.592 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:29 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2462180611' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:41:29.592 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:29 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-04-26T04:41:29.592 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:29 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2024-04-26T04:41:29.592 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:29 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-04-26T04:41:29.592 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:29 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-04-26T04:41:29.592 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:29 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2024-04-26T04:41:29.592 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:29 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-04-26T04:41:29.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:29 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:29.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:29 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2462180611' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:41:29.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:29 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-04-26T04:41:29.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:29 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2024-04-26T04:41:29.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:29 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-04-26T04:41:29.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:29 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-04-26T04:41:29.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:29 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2024-04-26T04:41:29.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:29 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-04-26T04:41:29.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:29 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:29.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:29 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2462180611' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:41:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:29 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-04-26T04:41:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:29 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2024-04-26T04:41:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:29 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-04-26T04:41:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:29 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-04-26T04:41:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:29 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2024-04-26T04:41:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:29 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-04-26T04:41:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:29 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:29.807 INFO:teuthology.orchestra.run.smithi156.stdout:55834574911 2024-04-26T04:41:30.423 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:30 smithi156 ceph-mon[39130]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:30.423 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:30 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-04-26T04:41:30.423 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:30 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2024-04-26T04:41:30.423 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:30 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-04-26T04:41:30.423 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:30 smithi156 ceph-mon[39130]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T04:41:30.423 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:30 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3726758987' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:41:30.424 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:30 smithi156 ceph-mon[27570]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:30.424 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:30 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-04-26T04:41:30.424 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:30 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2024-04-26T04:41:30.424 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:30 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-04-26T04:41:30.424 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:30 smithi156 ceph-mon[27570]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T04:41:30.424 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:30 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3726758987' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:41:30.722 INFO:teuthology.orchestra.run.smithi156.stdout:107374182453 2024-04-26T04:41:30.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:30 smithi171 ceph-mon[31139]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:30.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:30 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-04-26T04:41:30.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:30 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2024-04-26T04:41:30.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:30 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-04-26T04:41:30.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:30 smithi171 ceph-mon[31139]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T04:41:30.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:30 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3726758987' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:41:31.378 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182451 got 107374182453 for osd.3 2024-04-26T04:41:31.378 DEBUG:teuthology.parallel:result is None 2024-04-26T04:41:31.588 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:31 smithi156 ceph-mon[27570]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T04:41:31.588 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:31 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/552496732' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:41:31.588 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:31 smithi156 ceph-mon[39130]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T04:41:31.588 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:31 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/552496732' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:41:31.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:31 smithi171 ceph-mon[31139]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T04:41:31.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:31 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/552496732' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:41:32.561 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:32 smithi156 ceph-mon[27570]: pgmap v73: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:32.561 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:32 smithi156 ceph-mon[39130]: pgmap v73: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:32.567 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:32.590 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757289 got 163208757291 for osd.5 2024-04-26T04:41:32.590 DEBUG:teuthology.parallel:result is None 2024-04-26T04:41:32.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:32 smithi171 ceph-mon[31139]: pgmap v73: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:33.342 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574909 got 55834574911 for osd.1 2024-04-26T04:41:33.342 DEBUG:teuthology.parallel:result is None 2024-04-26T04:41:34.499 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:34 smithi156 ceph-mon[27570]: pgmap v74: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:34.499 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:34 smithi156 ceph-mon[39130]: pgmap v74: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:34.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:34 smithi171 ceph-mon[31139]: pgmap v74: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:34.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:41:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:41:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:41:34.799 INFO:teuthology.orchestra.run.smithi156.stdout:197568495655 2024-04-26T04:41:35.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:35 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2823610693' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:41:35.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:35 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2823610693' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:41:35.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:35 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2823610693' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:41:36.053 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495651 got 197568495655 for osd.6 2024-04-26T04:41:36.054 DEBUG:teuthology.parallel:result is None 2024-04-26T04:41:36.473 INFO:teuthology.orchestra.run.smithi156.stdout:231928234019 2024-04-26T04:41:36.563 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:41:36 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:41:36] "GET /metrics HTTP/1.1" 200 239051 "" "Prometheus/2.43.0" 2024-04-26T04:41:36.563 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:36 smithi156 ceph-mon[27570]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:36.564 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:36 smithi156 ceph-mon[39130]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:36.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:36 smithi171 ceph-mon[31139]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:37.461 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:37 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/90618639' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:41:37.462 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:37 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/90618639' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:41:37.486 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234015 got 231928234019 for osd.7 2024-04-26T04:41:37.486 DEBUG:teuthology.parallel:result is None 2024-04-26T04:41:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:37 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/90618639' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:41:37.922 INFO:teuthology.orchestra.run.smithi156.stdout:77309411387 2024-04-26T04:41:37.994 INFO:teuthology.orchestra.run.smithi156.stdout:34359738437 2024-04-26T04:41:38.402 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:38 smithi156 ceph-mon[27570]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:38.402 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:38 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/141708731' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:41:38.402 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:38 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1887008108' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:41:38.403 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:38 smithi156 ceph-mon[39130]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:38.403 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:38 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/141708731' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:41:38.403 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:38 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1887008108' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:41:38.666 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:38.681 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738433 got 34359738437 for osd.0 2024-04-26T04:41:38.682 DEBUG:teuthology.parallel:result is None 2024-04-26T04:41:38.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:38 smithi171 ceph-mon[31139]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:38.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:38 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/141708731' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:41:38.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:38 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1887008108' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:41:39.065 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411382 got 77309411387 for osd.2 2024-04-26T04:41:39.065 DEBUG:teuthology.parallel:result is None 2024-04-26T04:41:40.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:40 smithi156 ceph-mon[27570]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:40.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:40 smithi156 ceph-mon[39130]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:40.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:40 smithi171 ceph-mon[31139]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:41.160 INFO:teuthology.orchestra.run.smithi156.stdout:128849018930 2024-04-26T04:41:41.461 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:41 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1109772184' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:41:41.462 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:41 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1109772184' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:41:41.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:41 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1109772184' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:41:41.886 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018927 got 128849018930 for osd.4 2024-04-26T04:41:41.887 DEBUG:teuthology.parallel:result is None 2024-04-26T04:41:41.887 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-26T04:41:41.887 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:41:42.140 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:42.539 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:42 smithi156 ceph-mon[27570]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:42.540 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:42 smithi156 ceph-mon[39130]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:42.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:42 smithi171 ceph-mon[31139]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:44.082 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:41:44.083 INFO:teuthology.orchestra.run.smithi156.stderr:dumped all 2024-04-26T04:41:44.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:44 smithi156 ceph-mon[27570]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:44.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:44 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:44.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:44 smithi156 ceph-mon[39130]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:44.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:44 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:44.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:44 smithi171 ceph-mon[31139]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:44.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:44 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:44.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:41:44 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:41:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:41:44.766 INFO:teuthology.orchestra.run.smithi156.stdout:{"pg_ready":true,"pg_map":{"version":79,"stamp":"2024-04-26T04:41:43.219424+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386200,"kb_used_data":7332,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575016,"statfs":{"total":767960285184,"available":765516816384,"internally_reserved":0,"allocated":7507968,"data_stored":4228834,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002834"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.297568+0000","last_change":"2024-04-26T04:40:44.295432+0000","last_active":"2024-04-26T04:41:29.297568+0000","last_peered":"2024-04-26T04:41:29.297568+0000","last_clean":"2024-04-26T04:41:29.297568+0000","last_became_active":"2024-04-26T04:40:44.295146+0000","last_became_peered":"2024-04-26T04:40:44.295146+0000","last_unstale":"2024-04-26T04:41:29.297568+0000","last_undegraded":"2024-04-26T04:41:29.297568+0000","last_fullsized":"2024-04-26T04:41:29.297568+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:53:42.927961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457134+0000","last_change":"2024-04-26T04:40:44.269868+0000","last_active":"2024-04-26T04:41:29.457134+0000","last_peered":"2024-04-26T04:41:29.457134+0000","last_clean":"2024-04-26T04:41:29.457134+0000","last_became_active":"2024-04-26T04:40:44.269581+0000","last_became_peered":"2024-04-26T04:40:44.269581+0000","last_unstale":"2024-04-26T04:41:29.457134+0000","last_undegraded":"2024-04-26T04:41:29.457134+0000","last_fullsized":"2024-04-26T04:41:29.457134+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:57:53.521833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.293926+0000","last_change":"2024-04-26T04:40:44.293680+0000","last_active":"2024-04-26T04:41:29.293926+0000","last_peered":"2024-04-26T04:41:29.293926+0000","last_clean":"2024-04-26T04:41:29.293926+0000","last_became_active":"2024-04-26T04:40:44.293427+0000","last_became_peered":"2024-04-26T04:40:44.293427+0000","last_unstale":"2024-04-26T04:41:29.293926+0000","last_undegraded":"2024-04-26T04:41:29.293926+0000","last_fullsized":"2024-04-26T04:41:29.293926+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:29:58.842791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294458+0000","last_change":"2024-04-26T04:40:44.266052+0000","last_active":"2024-04-26T04:41:29.294458+0000","last_peered":"2024-04-26T04:41:29.294458+0000","last_clean":"2024-04-26T04:41:29.294458+0000","last_became_active":"2024-04-26T04:40:44.265778+0000","last_became_peered":"2024-04-26T04:40:44.265778+0000","last_unstale":"2024-04-26T04:41:29.294458+0000","last_undegraded":"2024-04-26T04:41:29.294458+0000","last_fullsized":"2024-04-26T04:41:29.294458+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:20:20.459628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.298607+0000","last_change":"2024-04-26T04:40:44.296114+0000","last_active":"2024-04-26T04:41:29.298607+0000","last_peered":"2024-04-26T04:41:29.298607+0000","last_clean":"2024-04-26T04:41:29.298607+0000","last_became_active":"2024-04-26T04:40:44.295707+0000","last_became_peered":"2024-04-26T04:40:44.295707+0000","last_unstale":"2024-04-26T04:41:29.298607+0000","last_undegraded":"2024-04-26T04:41:29.298607+0000","last_fullsized":"2024-04-26T04:41:29.298607+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:59:16.393307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294403+0000","last_change":"2024-04-26T04:40:44.269169+0000","last_active":"2024-04-26T04:41:29.294403+0000","last_peered":"2024-04-26T04:41:29.294403+0000","last_clean":"2024-04-26T04:41:29.294403+0000","last_became_active":"2024-04-26T04:40:44.268903+0000","last_became_peered":"2024-04-26T04:40:44.268903+0000","last_unstale":"2024-04-26T04:41:29.294403+0000","last_undegraded":"2024-04-26T04:41:29.294403+0000","last_fullsized":"2024-04-26T04:41:29.294403+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:07:22.671960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.298044+0000","last_change":"2024-04-26T04:40:44.295815+0000","last_active":"2024-04-26T04:41:29.298044+0000","last_peered":"2024-04-26T04:41:29.298044+0000","last_clean":"2024-04-26T04:41:29.298044+0000","last_became_active":"2024-04-26T04:40:44.295501+0000","last_became_peered":"2024-04-26T04:40:44.295501+0000","last_unstale":"2024-04-26T04:41:29.298044+0000","last_undegraded":"2024-04-26T04:41:29.298044+0000","last_fullsized":"2024-04-26T04:41:29.298044+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:21:41.810593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294268+0000","last_change":"2024-04-26T04:40:44.270149+0000","last_active":"2024-04-26T04:41:29.294268+0000","last_peered":"2024-04-26T04:41:29.294268+0000","last_clean":"2024-04-26T04:41:29.294268+0000","last_became_active":"2024-04-26T04:40:44.269847+0000","last_became_peered":"2024-04-26T04:40:44.269847+0000","last_unstale":"2024-04-26T04:41:29.294268+0000","last_undegraded":"2024-04-26T04:41:29.294268+0000","last_fullsized":"2024-04-26T04:41:29.294268+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:28:48.656498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.295874+0000","last_change":"2024-04-26T04:40:44.294836+0000","last_active":"2024-04-26T04:41:29.295874+0000","last_peered":"2024-04-26T04:41:29.295874+0000","last_clean":"2024-04-26T04:41:29.295874+0000","last_became_active":"2024-04-26T04:40:44.294525+0000","last_became_peered":"2024-04-26T04:40:44.294525+0000","last_unstale":"2024-04-26T04:41:29.295874+0000","last_undegraded":"2024-04-26T04:41:29.295874+0000","last_fullsized":"2024-04-26T04:41:29.295874+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:48:08.301019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.456975+0000","last_change":"2024-04-26T04:40:44.272691+0000","last_active":"2024-04-26T04:41:29.456975+0000","last_peered":"2024-04-26T04:41:29.456975+0000","last_clean":"2024-04-26T04:41:29.456975+0000","last_became_active":"2024-04-26T04:40:44.272339+0000","last_became_peered":"2024-04-26T04:40:44.272339+0000","last_unstale":"2024-04-26T04:41:29.456975+0000","last_undegraded":"2024-04-26T04:41:29.456975+0000","last_fullsized":"2024-04-26T04:41:29.456975+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:54:52.047487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.297926+0000","last_change":"2024-04-26T04:40:44.288207+0000","last_active":"2024-04-26T04:41:29.297926+0000","last_peered":"2024-04-26T04:41:29.297926+0000","last_clean":"2024-04-26T04:41:29.297926+0000","last_became_active":"2024-04-26T04:40:44.287939+0000","last_became_peered":"2024-04-26T04:40:44.287939+0000","last_unstale":"2024-04-26T04:41:29.297926+0000","last_undegraded":"2024-04-26T04:41:29.297926+0000","last_fullsized":"2024-04-26T04:41:29.297926+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:20:35.276921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457064+0000","last_change":"2024-04-26T04:40:44.275233+0000","last_active":"2024-04-26T04:41:29.457064+0000","last_peered":"2024-04-26T04:41:29.457064+0000","last_clean":"2024-04-26T04:41:29.457064+0000","last_became_active":"2024-04-26T04:40:44.274961+0000","last_became_peered":"2024-04-26T04:40:44.274961+0000","last_unstale":"2024-04-26T04:41:29.457064+0000","last_undegraded":"2024-04-26T04:41:29.457064+0000","last_fullsized":"2024-04-26T04:41:29.457064+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:10:05.950012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.300489+0000","last_change":"2024-04-26T04:39:29.193470+0000","last_active":"2024-04-26T04:41:29.300489+0000","last_peered":"2024-04-26T04:41:29.300489+0000","last_clean":"2024-04-26T04:41:29.300489+0000","last_became_active":"2024-04-26T04:39:29.193156+0000","last_became_peered":"2024-04-26T04:39:29.193156+0000","last_unstale":"2024-04-26T04:41:29.300489+0000","last_undegraded":"2024-04-26T04:41:29.300489+0000","last_fullsized":"2024-04-26T04:41:29.300489+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:16:39.309651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294192+0000","last_change":"2024-04-26T04:40:44.865475+0000","last_active":"2024-04-26T04:41:29.294192+0000","last_peered":"2024-04-26T04:41:29.294192+0000","last_clean":"2024-04-26T04:41:29.294192+0000","last_became_active":"2024-04-26T04:40:44.865022+0000","last_became_peered":"2024-04-26T04:40:44.865022+0000","last_unstale":"2024-04-26T04:41:29.294192+0000","last_undegraded":"2024-04-26T04:41:29.294192+0000","last_fullsized":"2024-04-26T04:41:29.294192+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:37:57.242315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.295727+0000","last_change":"2024-04-26T04:39:29.193876+0000","last_active":"2024-04-26T04:41:29.295727+0000","last_peered":"2024-04-26T04:41:29.295727+0000","last_clean":"2024-04-26T04:41:29.295727+0000","last_became_active":"2024-04-26T04:39:29.193532+0000","last_became_peered":"2024-04-26T04:39:29.193532+0000","last_unstale":"2024-04-26T04:41:29.295727+0000","last_undegraded":"2024-04-26T04:41:29.295727+0000","last_fullsized":"2024-04-26T04:41:29.295727+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:58:26.728897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.295311+0000","last_change":"2024-04-26T04:40:44.269328+0000","last_active":"2024-04-26T04:41:29.295311+0000","last_peered":"2024-04-26T04:41:29.295311+0000","last_clean":"2024-04-26T04:41:29.295311+0000","last_became_active":"2024-04-26T04:40:44.269047+0000","last_became_peered":"2024-04-26T04:40:44.269047+0000","last_unstale":"2024-04-26T04:41:29.295311+0000","last_undegraded":"2024-04-26T04:41:29.295311+0000","last_fullsized":"2024-04-26T04:41:29.295311+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:30:56.122947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.456829+0000","last_change":"2024-04-26T04:40:44.865161+0000","last_active":"2024-04-26T04:41:29.456829+0000","last_peered":"2024-04-26T04:41:29.456829+0000","last_clean":"2024-04-26T04:41:29.456829+0000","last_became_active":"2024-04-26T04:40:44.864770+0000","last_became_peered":"2024-04-26T04:40:44.864770+0000","last_unstale":"2024-04-26T04:41:29.456829+0000","last_undegraded":"2024-04-26T04:41:29.456829+0000","last_fullsized":"2024-04-26T04:41:29.456829+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:10:58.140636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.295554+0000","last_change":"2024-04-26T04:40:44.264938+0000","last_active":"2024-04-26T04:41:29.295554+0000","last_peered":"2024-04-26T04:41:29.295554+0000","last_clean":"2024-04-26T04:41:29.295554+0000","last_became_active":"2024-04-26T04:40:44.264670+0000","last_became_peered":"2024-04-26T04:40:44.264670+0000","last_unstale":"2024-04-26T04:41:29.295554+0000","last_undegraded":"2024-04-26T04:41:29.295554+0000","last_fullsized":"2024-04-26T04:41:29.295554+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:21:05.870793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457679+0000","last_change":"2024-04-26T04:39:29.224943+0000","last_active":"2024-04-26T04:41:29.457679+0000","last_peered":"2024-04-26T04:41:29.457679+0000","last_clean":"2024-04-26T04:41:29.457679+0000","last_became_active":"2024-04-26T04:39:29.224586+0000","last_became_peered":"2024-04-26T04:39:29.224586+0000","last_unstale":"2024-04-26T04:41:29.457679+0000","last_undegraded":"2024-04-26T04:41:29.457679+0000","last_fullsized":"2024-04-26T04:41:29.457679+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:50:41.653796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457113+0000","last_change":"2024-04-26T04:40:44.293195+0000","last_active":"2024-04-26T04:41:29.457113+0000","last_peered":"2024-04-26T04:41:29.457113+0000","last_clean":"2024-04-26T04:41:29.457113+0000","last_became_active":"2024-04-26T04:40:44.292921+0000","last_became_peered":"2024-04-26T04:40:44.292921+0000","last_unstale":"2024-04-26T04:41:29.457113+0000","last_undegraded":"2024-04-26T04:41:29.457113+0000","last_fullsized":"2024-04-26T04:41:29.457113+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:31:15.459441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.456857+0000","last_change":"2024-04-26T04:40:44.270682+0000","last_active":"2024-04-26T04:41:29.456857+0000","last_peered":"2024-04-26T04:41:29.456857+0000","last_clean":"2024-04-26T04:41:29.456857+0000","last_became_active":"2024-04-26T04:40:44.270404+0000","last_became_peered":"2024-04-26T04:40:44.270404+0000","last_unstale":"2024-04-26T04:41:29.456857+0000","last_undegraded":"2024-04-26T04:41:29.456857+0000","last_fullsized":"2024-04-26T04:41:29.456857+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:48:44.848733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457413+0000","last_change":"2024-04-26T04:39:29.224807+0000","last_active":"2024-04-26T04:41:29.457413+0000","last_peered":"2024-04-26T04:41:29.457413+0000","last_clean":"2024-04-26T04:41:29.457413+0000","last_became_active":"2024-04-26T04:39:29.224497+0000","last_became_peered":"2024-04-26T04:39:29.224497+0000","last_unstale":"2024-04-26T04:41:29.457413+0000","last_undegraded":"2024-04-26T04:41:29.457413+0000","last_fullsized":"2024-04-26T04:41:29.457413+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:20:03.848049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.300470+0000","last_change":"2024-04-26T04:40:44.864165+0000","last_active":"2024-04-26T04:41:29.300470+0000","last_peered":"2024-04-26T04:41:29.300470+0000","last_clean":"2024-04-26T04:41:29.300470+0000","last_became_active":"2024-04-26T04:40:44.863874+0000","last_became_peered":"2024-04-26T04:40:44.863874+0000","last_unstale":"2024-04-26T04:41:29.300470+0000","last_undegraded":"2024-04-26T04:41:29.300470+0000","last_fullsized":"2024-04-26T04:41:29.300470+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:32:35.821759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.293847+0000","last_change":"2024-04-26T04:40:44.265221+0000","last_active":"2024-04-26T04:41:29.293847+0000","last_peered":"2024-04-26T04:41:29.293847+0000","last_clean":"2024-04-26T04:41:29.293847+0000","last_became_active":"2024-04-26T04:40:44.264934+0000","last_became_peered":"2024-04-26T04:40:44.264934+0000","last_unstale":"2024-04-26T04:41:29.293847+0000","last_undegraded":"2024-04-26T04:41:29.293847+0000","last_fullsized":"2024-04-26T04:41:29.293847+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:56:43.564670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"62'128","reported_seq":276,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457373+0000","last_change":"2024-04-26T04:39:00.628070+0000","last_active":"2024-04-26T04:41:29.457373+0000","last_peered":"2024-04-26T04:41:29.457373+0000","last_clean":"2024-04-26T04:41:29.457373+0000","last_became_active":"2024-04-26T04:39:00.627546+0000","last_became_peered":"2024-04-26T04:39:00.627546+0000","last_unstale":"2024-04-26T04:41:29.457373+0000","last_undegraded":"2024-04-26T04:41:29.457373+0000","last_fullsized":"2024-04-26T04:41:29.457373+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:36:55.213599+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:36:55.213599+0000","last_clean_scrub_stamp":"2024-04-26T04:36:55.213599+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:26:44.288000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294121+0000","last_change":"2024-04-26T04:39:30.840097+0000","last_active":"2024-04-26T04:41:29.294121+0000","last_peered":"2024-04-26T04:41:29.294121+0000","last_clean":"2024-04-26T04:41:29.294121+0000","last_became_active":"2024-04-26T04:39:30.839750+0000","last_became_peered":"2024-04-26T04:39:30.839750+0000","last_unstale":"2024-04-26T04:41:29.294121+0000","last_undegraded":"2024-04-26T04:41:29.294121+0000","last_fullsized":"2024-04-26T04:41:29.294121+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:00:14.008690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294850+0000","last_change":"2024-04-26T04:40:44.299462+0000","last_active":"2024-04-26T04:41:29.294850+0000","last_peered":"2024-04-26T04:41:29.294850+0000","last_clean":"2024-04-26T04:41:29.294850+0000","last_became_active":"2024-04-26T04:40:44.298852+0000","last_became_peered":"2024-04-26T04:40:44.298852+0000","last_unstale":"2024-04-26T04:41:29.294850+0000","last_undegraded":"2024-04-26T04:41:29.294850+0000","last_fullsized":"2024-04-26T04:41:29.294850+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:37:55.693467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294042+0000","last_change":"2024-04-26T04:40:44.265714+0000","last_active":"2024-04-26T04:41:29.294042+0000","last_peered":"2024-04-26T04:41:29.294042+0000","last_clean":"2024-04-26T04:41:29.294042+0000","last_became_active":"2024-04-26T04:40:44.265299+0000","last_became_peered":"2024-04-26T04:40:44.265299+0000","last_unstale":"2024-04-26T04:41:29.294042+0000","last_undegraded":"2024-04-26T04:41:29.294042+0000","last_fullsized":"2024-04-26T04:41:29.294042+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:49:33.627495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.296137+0000","last_change":"2024-04-26T04:39:29.224560+0000","last_active":"2024-04-26T04:41:29.296137+0000","last_peered":"2024-04-26T04:41:29.296137+0000","last_clean":"2024-04-26T04:41:29.296137+0000","last_became_active":"2024-04-26T04:39:29.224239+0000","last_became_peered":"2024-04-26T04:39:29.224239+0000","last_unstale":"2024-04-26T04:41:29.296137+0000","last_undegraded":"2024-04-26T04:41:29.296137+0000","last_fullsized":"2024-04-26T04:41:29.296137+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:27:28.097185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.295488+0000","last_change":"2024-04-26T04:40:44.297277+0000","last_active":"2024-04-26T04:41:29.295488+0000","last_peered":"2024-04-26T04:41:29.295488+0000","last_clean":"2024-04-26T04:41:29.295488+0000","last_became_active":"2024-04-26T04:40:44.295885+0000","last_became_peered":"2024-04-26T04:40:44.295885+0000","last_unstale":"2024-04-26T04:41:29.295488+0000","last_undegraded":"2024-04-26T04:41:29.295488+0000","last_fullsized":"2024-04-26T04:41:29.295488+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:07:25.143389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.297839+0000","last_change":"2024-04-26T04:40:44.271779+0000","last_active":"2024-04-26T04:41:29.297839+0000","last_peered":"2024-04-26T04:41:29.297839+0000","last_clean":"2024-04-26T04:41:29.297839+0000","last_became_active":"2024-04-26T04:40:44.270674+0000","last_became_peered":"2024-04-26T04:40:44.270674+0000","last_unstale":"2024-04-26T04:41:29.297839+0000","last_undegraded":"2024-04-26T04:41:29.297839+0000","last_fullsized":"2024-04-26T04:41:29.297839+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:37:12.036437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294007+0000","last_change":"2024-04-26T04:39:30.545245+0000","last_active":"2024-04-26T04:41:29.294007+0000","last_peered":"2024-04-26T04:41:29.294007+0000","last_clean":"2024-04-26T04:41:29.294007+0000","last_became_active":"2024-04-26T04:39:30.544940+0000","last_became_peered":"2024-04-26T04:39:30.544940+0000","last_unstale":"2024-04-26T04:41:29.294007+0000","last_undegraded":"2024-04-26T04:41:29.294007+0000","last_fullsized":"2024-04-26T04:41:29.294007+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:53:32.749890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:41:30.299248+0000","last_change":"2024-04-26T04:41:30.299248+0000","last_active":"2024-04-26T04:41:30.299248+0000","last_peered":"2024-04-26T04:41:30.299248+0000","last_clean":"2024-04-26T04:41:30.299248+0000","last_became_active":"2024-04-26T04:41:30.299009+0000","last_became_peered":"2024-04-26T04:41:30.299009+0000","last_unstale":"2024-04-26T04:41:30.299248+0000","last_undegraded":"2024-04-26T04:41:30.299248+0000","last_fullsized":"2024-04-26T04:41:30.299248+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:29:16.315075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294416+0000","last_change":"2024-04-26T04:40:44.265375+0000","last_active":"2024-04-26T04:41:29.294416+0000","last_peered":"2024-04-26T04:41:29.294416+0000","last_clean":"2024-04-26T04:41:29.294416+0000","last_became_active":"2024-04-26T04:40:44.265077+0000","last_became_peered":"2024-04-26T04:40:44.265077+0000","last_unstale":"2024-04-26T04:41:29.294416+0000","last_undegraded":"2024-04-26T04:41:29.294416+0000","last_fullsized":"2024-04-26T04:41:29.294416+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:37:54.273375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.295422+0000","last_change":"2024-04-26T04:39:29.224546+0000","last_active":"2024-04-26T04:41:29.295422+0000","last_peered":"2024-04-26T04:41:29.295422+0000","last_clean":"2024-04-26T04:41:29.295422+0000","last_became_active":"2024-04-26T04:39:29.224238+0000","last_became_peered":"2024-04-26T04:39:29.224238+0000","last_unstale":"2024-04-26T04:41:29.295422+0000","last_undegraded":"2024-04-26T04:41:29.295422+0000","last_fullsized":"2024-04-26T04:41:29.295422+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:45:24.471564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:41:30.301211+0000","last_change":"2024-04-26T04:41:30.301211+0000","last_active":"2024-04-26T04:41:30.301211+0000","last_peered":"2024-04-26T04:41:30.301211+0000","last_clean":"2024-04-26T04:41:30.301211+0000","last_became_active":"2024-04-26T04:41:30.300881+0000","last_became_peered":"2024-04-26T04:41:30.300881+0000","last_unstale":"2024-04-26T04:41:30.301211+0000","last_undegraded":"2024-04-26T04:41:30.301211+0000","last_fullsized":"2024-04-26T04:41:30.301211+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:28:09.438146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.293485+0000","last_change":"2024-04-26T04:40:44.263300+0000","last_active":"2024-04-26T04:41:29.293485+0000","last_peered":"2024-04-26T04:41:29.293485+0000","last_clean":"2024-04-26T04:41:29.293485+0000","last_became_active":"2024-04-26T04:40:44.263005+0000","last_became_peered":"2024-04-26T04:40:44.263005+0000","last_unstale":"2024-04-26T04:41:29.293485+0000","last_undegraded":"2024-04-26T04:41:29.293485+0000","last_fullsized":"2024-04-26T04:41:29.293485+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:45:56.407034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":22,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:41:30.559476+0000","last_change":"2024-04-26T04:41:30.559476+0000","last_active":"2024-04-26T04:41:30.559476+0000","last_peered":"2024-04-26T04:41:30.559476+0000","last_clean":"2024-04-26T04:41:30.559476+0000","last_became_active":"2024-04-26T04:41:30.559052+0000","last_became_peered":"2024-04-26T04:41:30.559052+0000","last_unstale":"2024-04-26T04:41:30.559476+0000","last_undegraded":"2024-04-26T04:41:30.559476+0000","last_fullsized":"2024-04-26T04:41:30.559476+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:56:25.173257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.297964+0000","last_change":"2024-04-26T04:40:44.270029+0000","last_active":"2024-04-26T04:41:29.297964+0000","last_peered":"2024-04-26T04:41:29.297964+0000","last_clean":"2024-04-26T04:41:29.297964+0000","last_became_active":"2024-04-26T04:40:44.268568+0000","last_became_peered":"2024-04-26T04:40:44.268568+0000","last_unstale":"2024-04-26T04:41:29.297964+0000","last_undegraded":"2024-04-26T04:41:29.297964+0000","last_fullsized":"2024-04-26T04:41:29.297964+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:39:19.144986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.298579+0000","last_change":"2024-04-26T04:40:44.294654+0000","last_active":"2024-04-26T04:41:29.298579+0000","last_peered":"2024-04-26T04:41:29.298579+0000","last_clean":"2024-04-26T04:41:29.298579+0000","last_became_active":"2024-04-26T04:40:44.294362+0000","last_became_peered":"2024-04-26T04:40:44.294362+0000","last_unstale":"2024-04-26T04:41:29.298579+0000","last_undegraded":"2024-04-26T04:41:29.298579+0000","last_fullsized":"2024-04-26T04:41:29.298579+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:20:19.295674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.300429+0000","last_change":"2024-04-26T04:40:44.274083+0000","last_active":"2024-04-26T04:41:29.300429+0000","last_peered":"2024-04-26T04:41:29.300429+0000","last_clean":"2024-04-26T04:41:29.300429+0000","last_became_active":"2024-04-26T04:40:44.273811+0000","last_became_peered":"2024-04-26T04:40:44.273811+0000","last_unstale":"2024-04-26T04:41:29.300429+0000","last_undegraded":"2024-04-26T04:41:29.300429+0000","last_fullsized":"2024-04-26T04:41:29.300429+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:33:49.201206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.298151+0000","last_change":"2024-04-26T04:40:44.296732+0000","last_active":"2024-04-26T04:41:29.298151+0000","last_peered":"2024-04-26T04:41:29.298151+0000","last_clean":"2024-04-26T04:41:29.298151+0000","last_became_active":"2024-04-26T04:40:44.296414+0000","last_became_peered":"2024-04-26T04:40:44.296414+0000","last_unstale":"2024-04-26T04:41:29.298151+0000","last_undegraded":"2024-04-26T04:41:29.298151+0000","last_fullsized":"2024-04-26T04:41:29.298151+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:55:50.262968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294052+0000","last_change":"2024-04-26T04:40:44.274301+0000","last_active":"2024-04-26T04:41:29.294052+0000","last_peered":"2024-04-26T04:41:29.294052+0000","last_clean":"2024-04-26T04:41:29.294052+0000","last_became_active":"2024-04-26T04:40:44.273989+0000","last_became_peered":"2024-04-26T04:40:44.273989+0000","last_unstale":"2024-04-26T04:41:29.294052+0000","last_undegraded":"2024-04-26T04:41:29.294052+0000","last_fullsized":"2024-04-26T04:41:29.294052+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:09:09.093513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.456974+0000","last_change":"2024-04-26T04:40:44.292554+0000","last_active":"2024-04-26T04:41:29.456974+0000","last_peered":"2024-04-26T04:41:29.456974+0000","last_clean":"2024-04-26T04:41:29.456974+0000","last_became_active":"2024-04-26T04:40:44.292257+0000","last_became_peered":"2024-04-26T04:40:44.292257+0000","last_unstale":"2024-04-26T04:41:29.456974+0000","last_undegraded":"2024-04-26T04:41:29.456974+0000","last_fullsized":"2024-04-26T04:41:29.456974+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:28:16.184538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294147+0000","last_change":"2024-04-26T04:40:44.267065+0000","last_active":"2024-04-26T04:41:29.294147+0000","last_peered":"2024-04-26T04:41:29.294147+0000","last_clean":"2024-04-26T04:41:29.294147+0000","last_became_active":"2024-04-26T04:40:44.266787+0000","last_became_peered":"2024-04-26T04:40:44.266787+0000","last_unstale":"2024-04-26T04:41:29.294147+0000","last_undegraded":"2024-04-26T04:41:29.294147+0000","last_fullsized":"2024-04-26T04:41:29.294147+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:42:02.342804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.300326+0000","last_change":"2024-04-26T04:40:44.864885+0000","last_active":"2024-04-26T04:41:29.300326+0000","last_peered":"2024-04-26T04:41:29.300326+0000","last_clean":"2024-04-26T04:41:29.300326+0000","last_became_active":"2024-04-26T04:40:44.864612+0000","last_became_peered":"2024-04-26T04:40:44.864612+0000","last_unstale":"2024-04-26T04:41:29.300326+0000","last_undegraded":"2024-04-26T04:41:29.300326+0000","last_fullsized":"2024-04-26T04:41:29.300326+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:31:17.764915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457789+0000","last_change":"2024-04-26T04:40:44.265256+0000","last_active":"2024-04-26T04:41:29.457789+0000","last_peered":"2024-04-26T04:41:29.457789+0000","last_clean":"2024-04-26T04:41:29.457789+0000","last_became_active":"2024-04-26T04:40:44.264777+0000","last_became_peered":"2024-04-26T04:40:44.264777+0000","last_unstale":"2024-04-26T04:41:29.457789+0000","last_undegraded":"2024-04-26T04:41:29.457789+0000","last_fullsized":"2024-04-26T04:41:29.457789+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:05:04.374938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.297761+0000","last_change":"2024-04-26T04:40:44.296722+0000","last_active":"2024-04-26T04:41:29.297761+0000","last_peered":"2024-04-26T04:41:29.297761+0000","last_clean":"2024-04-26T04:41:29.297761+0000","last_became_active":"2024-04-26T04:40:44.296258+0000","last_became_peered":"2024-04-26T04:40:44.296258+0000","last_unstale":"2024-04-26T04:41:29.297761+0000","last_undegraded":"2024-04-26T04:41:29.297761+0000","last_fullsized":"2024-04-26T04:41:29.297761+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:18:01.299512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457803+0000","last_change":"2024-04-26T04:40:44.274325+0000","last_active":"2024-04-26T04:41:29.457803+0000","last_peered":"2024-04-26T04:41:29.457803+0000","last_clean":"2024-04-26T04:41:29.457803+0000","last_became_active":"2024-04-26T04:40:44.274029+0000","last_became_peered":"2024-04-26T04:40:44.274029+0000","last_unstale":"2024-04-26T04:41:29.457803+0000","last_undegraded":"2024-04-26T04:41:29.457803+0000","last_fullsized":"2024-04-26T04:41:29.457803+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:02:00.292982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.293996+0000","last_change":"2024-04-26T04:40:44.864902+0000","last_active":"2024-04-26T04:41:29.293996+0000","last_peered":"2024-04-26T04:41:29.293996+0000","last_clean":"2024-04-26T04:41:29.293996+0000","last_became_active":"2024-04-26T04:40:44.864603+0000","last_became_peered":"2024-04-26T04:40:44.864603+0000","last_unstale":"2024-04-26T04:41:29.293996+0000","last_undegraded":"2024-04-26T04:41:29.293996+0000","last_fullsized":"2024-04-26T04:41:29.293996+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:50:44.169231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457856+0000","last_change":"2024-04-26T04:40:44.269258+0000","last_active":"2024-04-26T04:41:29.457856+0000","last_peered":"2024-04-26T04:41:29.457856+0000","last_clean":"2024-04-26T04:41:29.457856+0000","last_became_active":"2024-04-26T04:40:44.268931+0000","last_became_peered":"2024-04-26T04:40:44.268931+0000","last_unstale":"2024-04-26T04:41:29.457856+0000","last_undegraded":"2024-04-26T04:41:29.457856+0000","last_fullsized":"2024-04-26T04:41:29.457856+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:51:21.554999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.295395+0000","last_change":"2024-04-26T04:40:44.864375+0000","last_active":"2024-04-26T04:41:29.295395+0000","last_peered":"2024-04-26T04:41:29.295395+0000","last_clean":"2024-04-26T04:41:29.295395+0000","last_became_active":"2024-04-26T04:40:44.864045+0000","last_became_peered":"2024-04-26T04:40:44.864045+0000","last_unstale":"2024-04-26T04:41:29.295395+0000","last_undegraded":"2024-04-26T04:41:29.295395+0000","last_fullsized":"2024-04-26T04:41:29.295395+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:44:41.278510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":21,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.300237+0000","last_change":"2024-04-26T04:40:44.272426+0000","last_active":"2024-04-26T04:41:29.300237+0000","last_peered":"2024-04-26T04:41:29.300237+0000","last_clean":"2024-04-26T04:41:29.300237+0000","last_became_active":"2024-04-26T04:40:44.272127+0000","last_became_peered":"2024-04-26T04:40:44.272127+0000","last_unstale":"2024-04-26T04:41:29.300237+0000","last_undegraded":"2024-04-26T04:41:29.300237+0000","last_fullsized":"2024-04-26T04:41:29.300237+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:16:45.181003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.456952+0000","last_change":"2024-04-26T04:40:44.865102+0000","last_active":"2024-04-26T04:41:29.456952+0000","last_peered":"2024-04-26T04:41:29.456952+0000","last_clean":"2024-04-26T04:41:29.456952+0000","last_became_active":"2024-04-26T04:40:44.864785+0000","last_became_peered":"2024-04-26T04:40:44.864785+0000","last_unstale":"2024-04-26T04:41:29.456952+0000","last_undegraded":"2024-04-26T04:41:29.456952+0000","last_fullsized":"2024-04-26T04:41:29.456952+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:12:42.433794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457452+0000","last_change":"2024-04-26T04:40:44.269467+0000","last_active":"2024-04-26T04:41:29.457452+0000","last_peered":"2024-04-26T04:41:29.457452+0000","last_clean":"2024-04-26T04:41:29.457452+0000","last_became_active":"2024-04-26T04:40:44.269173+0000","last_became_peered":"2024-04-26T04:40:44.269173+0000","last_unstale":"2024-04-26T04:41:29.457452+0000","last_undegraded":"2024-04-26T04:41:29.457452+0000","last_fullsized":"2024-04-26T04:41:29.457452+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:17:31.868820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294288+0000","last_change":"2024-04-26T04:40:44.864213+0000","last_active":"2024-04-26T04:41:29.294288+0000","last_peered":"2024-04-26T04:41:29.294288+0000","last_clean":"2024-04-26T04:41:29.294288+0000","last_became_active":"2024-04-26T04:40:44.863912+0000","last_became_peered":"2024-04-26T04:40:44.863912+0000","last_unstale":"2024-04-26T04:41:29.294288+0000","last_undegraded":"2024-04-26T04:41:29.294288+0000","last_fullsized":"2024-04-26T04:41:29.294288+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:38:46.486109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.456901+0000","last_change":"2024-04-26T04:40:44.271893+0000","last_active":"2024-04-26T04:41:29.456901+0000","last_peered":"2024-04-26T04:41:29.456901+0000","last_clean":"2024-04-26T04:41:29.456901+0000","last_became_active":"2024-04-26T04:40:44.271613+0000","last_became_peered":"2024-04-26T04:40:44.271613+0000","last_unstale":"2024-04-26T04:41:29.456901+0000","last_undegraded":"2024-04-26T04:41:29.456901+0000","last_fullsized":"2024-04-26T04:41:29.456901+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:33:49.567831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294371+0000","last_change":"2024-04-26T04:40:44.864369+0000","last_active":"2024-04-26T04:41:29.294371+0000","last_peered":"2024-04-26T04:41:29.294371+0000","last_clean":"2024-04-26T04:41:29.294371+0000","last_became_active":"2024-04-26T04:40:44.864062+0000","last_became_peered":"2024-04-26T04:40:44.864062+0000","last_unstale":"2024-04-26T04:41:29.294371+0000","last_undegraded":"2024-04-26T04:41:29.294371+0000","last_fullsized":"2024-04-26T04:41:29.294371+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:21:41.835888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457645+0000","last_change":"2024-04-26T04:40:44.270046+0000","last_active":"2024-04-26T04:41:29.457645+0000","last_peered":"2024-04-26T04:41:29.457645+0000","last_clean":"2024-04-26T04:41:29.457645+0000","last_became_active":"2024-04-26T04:40:44.269776+0000","last_became_peered":"2024-04-26T04:40:44.269776+0000","last_unstale":"2024-04-26T04:41:29.457645+0000","last_undegraded":"2024-04-26T04:41:29.457645+0000","last_fullsized":"2024-04-26T04:41:29.457645+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:38:17.750585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.297895+0000","last_change":"2024-04-26T04:40:44.294891+0000","last_active":"2024-04-26T04:41:29.297895+0000","last_peered":"2024-04-26T04:41:29.297895+0000","last_clean":"2024-04-26T04:41:29.297895+0000","last_became_active":"2024-04-26T04:40:44.294556+0000","last_became_peered":"2024-04-26T04:40:44.294556+0000","last_unstale":"2024-04-26T04:41:29.297895+0000","last_undegraded":"2024-04-26T04:41:29.297895+0000","last_fullsized":"2024-04-26T04:41:29.297895+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:57:20.227140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"67'6","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.298198+0000","last_change":"2024-04-26T04:40:44.272084+0000","last_active":"2024-04-26T04:41:29.298198+0000","last_peered":"2024-04-26T04:41:29.298198+0000","last_clean":"2024-04-26T04:41:29.298198+0000","last_became_active":"2024-04-26T04:40:44.271806+0000","last_became_peered":"2024-04-26T04:40:44.271806+0000","last_unstale":"2024-04-26T04:41:29.298198+0000","last_undegraded":"2024-04-26T04:41:29.298198+0000","last_fullsized":"2024-04-26T04:41:29.298198+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:42:50.578965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294544+0000","last_change":"2024-04-26T04:40:44.299343+0000","last_active":"2024-04-26T04:41:29.294544+0000","last_peered":"2024-04-26T04:41:29.294544+0000","last_clean":"2024-04-26T04:41:29.294544+0000","last_became_active":"2024-04-26T04:40:44.299081+0000","last_became_peered":"2024-04-26T04:40:44.299081+0000","last_unstale":"2024-04-26T04:41:29.294544+0000","last_undegraded":"2024-04-26T04:41:29.294544+0000","last_fullsized":"2024-04-26T04:41:29.294544+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:46:17.439373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457656+0000","last_change":"2024-04-26T04:40:44.267055+0000","last_active":"2024-04-26T04:41:29.457656+0000","last_peered":"2024-04-26T04:41:29.457656+0000","last_clean":"2024-04-26T04:41:29.457656+0000","last_became_active":"2024-04-26T04:40:44.266753+0000","last_became_peered":"2024-04-26T04:40:44.266753+0000","last_unstale":"2024-04-26T04:41:29.457656+0000","last_undegraded":"2024-04-26T04:41:29.457656+0000","last_fullsized":"2024-04-26T04:41:29.457656+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:16:33.130853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.293956+0000","last_change":"2024-04-26T04:40:44.865760+0000","last_active":"2024-04-26T04:41:29.293956+0000","last_peered":"2024-04-26T04:41:29.293956+0000","last_clean":"2024-04-26T04:41:29.293956+0000","last_became_active":"2024-04-26T04:40:44.865464+0000","last_became_peered":"2024-04-26T04:40:44.865464+0000","last_unstale":"2024-04-26T04:41:29.293956+0000","last_undegraded":"2024-04-26T04:41:29.293956+0000","last_fullsized":"2024-04-26T04:41:29.293956+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:52:36.042764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294340+0000","last_change":"2024-04-26T04:40:44.274518+0000","last_active":"2024-04-26T04:41:29.294340+0000","last_peered":"2024-04-26T04:41:29.294340+0000","last_clean":"2024-04-26T04:41:29.294340+0000","last_became_active":"2024-04-26T04:40:44.274180+0000","last_became_peered":"2024-04-26T04:40:44.274180+0000","last_unstale":"2024-04-26T04:41:29.294340+0000","last_undegraded":"2024-04-26T04:41:29.294340+0000","last_fullsized":"2024-04-26T04:41:29.294340+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:16:23.377738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294271+0000","last_change":"2024-04-26T04:40:44.293162+0000","last_active":"2024-04-26T04:41:29.294271+0000","last_peered":"2024-04-26T04:41:29.294271+0000","last_clean":"2024-04-26T04:41:29.294271+0000","last_became_active":"2024-04-26T04:40:44.292910+0000","last_became_peered":"2024-04-26T04:40:44.292910+0000","last_unstale":"2024-04-26T04:41:29.294271+0000","last_undegraded":"2024-04-26T04:41:29.294271+0000","last_fullsized":"2024-04-26T04:41:29.294271+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:17:45.414448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457088+0000","last_change":"2024-04-26T04:40:44.271984+0000","last_active":"2024-04-26T04:41:29.457088+0000","last_peered":"2024-04-26T04:41:29.457088+0000","last_clean":"2024-04-26T04:41:29.457088+0000","last_became_active":"2024-04-26T04:40:44.271731+0000","last_became_peered":"2024-04-26T04:40:44.271731+0000","last_unstale":"2024-04-26T04:41:29.457088+0000","last_undegraded":"2024-04-26T04:41:29.457088+0000","last_fullsized":"2024-04-26T04:41:29.457088+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:44:05.726631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.300456+0000","last_change":"2024-04-26T04:40:44.294330+0000","last_active":"2024-04-26T04:41:29.300456+0000","last_peered":"2024-04-26T04:41:29.300456+0000","last_clean":"2024-04-26T04:41:29.300456+0000","last_became_active":"2024-04-26T04:40:44.294017+0000","last_became_peered":"2024-04-26T04:40:44.294017+0000","last_unstale":"2024-04-26T04:41:29.300456+0000","last_undegraded":"2024-04-26T04:41:29.300456+0000","last_fullsized":"2024-04-26T04:41:29.300456+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:24:45.587863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294349+0000","last_change":"2024-04-26T04:40:44.267340+0000","last_active":"2024-04-26T04:41:29.294349+0000","last_peered":"2024-04-26T04:41:29.294349+0000","last_clean":"2024-04-26T04:41:29.294349+0000","last_became_active":"2024-04-26T04:40:44.267083+0000","last_became_peered":"2024-04-26T04:40:44.267083+0000","last_unstale":"2024-04-26T04:41:29.294349+0000","last_undegraded":"2024-04-26T04:41:29.294349+0000","last_fullsized":"2024-04-26T04:41:29.294349+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:05:59.489560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457057+0000","last_change":"2024-04-26T04:40:44.865222+0000","last_active":"2024-04-26T04:41:29.457057+0000","last_peered":"2024-04-26T04:41:29.457057+0000","last_clean":"2024-04-26T04:41:29.457057+0000","last_became_active":"2024-04-26T04:40:44.864827+0000","last_became_peered":"2024-04-26T04:40:44.864827+0000","last_unstale":"2024-04-26T04:41:29.457057+0000","last_undegraded":"2024-04-26T04:41:29.457057+0000","last_fullsized":"2024-04-26T04:41:29.457057+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:18:54.156267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.293775+0000","last_change":"2024-04-26T04:40:44.274169+0000","last_active":"2024-04-26T04:41:29.293775+0000","last_peered":"2024-04-26T04:41:29.293775+0000","last_clean":"2024-04-26T04:41:29.293775+0000","last_became_active":"2024-04-26T04:40:44.273904+0000","last_became_peered":"2024-04-26T04:40:44.273904+0000","last_unstale":"2024-04-26T04:41:29.293775+0000","last_undegraded":"2024-04-26T04:41:29.293775+0000","last_fullsized":"2024-04-26T04:41:29.293775+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:43:11.200334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294246+0000","last_change":"2024-04-26T04:40:44.293412+0000","last_active":"2024-04-26T04:41:29.294246+0000","last_peered":"2024-04-26T04:41:29.294246+0000","last_clean":"2024-04-26T04:41:29.294246+0000","last_became_active":"2024-04-26T04:40:44.293111+0000","last_became_peered":"2024-04-26T04:40:44.293111+0000","last_unstale":"2024-04-26T04:41:29.294246+0000","last_undegraded":"2024-04-26T04:41:29.294246+0000","last_fullsized":"2024-04-26T04:41:29.294246+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:11:55.279941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.298493+0000","last_change":"2024-04-26T04:40:44.268160+0000","last_active":"2024-04-26T04:41:29.298493+0000","last_peered":"2024-04-26T04:41:29.298493+0000","last_clean":"2024-04-26T04:41:29.298493+0000","last_became_active":"2024-04-26T04:40:44.267832+0000","last_became_peered":"2024-04-26T04:40:44.267832+0000","last_unstale":"2024-04-26T04:41:29.298493+0000","last_undegraded":"2024-04-26T04:41:29.298493+0000","last_fullsized":"2024-04-26T04:41:29.298493+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:44:23.755059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":5,"up_from":38,"seq":163208757294,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":696,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":712704,"data_stored":307054,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:41:09 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.51200000000000001,"15min":0.50800000000000001},"min":{"1min":0.311,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.75,"5min":0.877,"15min":0.877},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.502,"15min":0.49099999999999999},"min":{"1min":0.32800000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.66600000000000004,"5min":0.747,"15min":0.747},"last":0.61699999999999999}]},{"osd":1,"last update":"Fri Apr 26 04:41:09 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.52900000000000003,"15min":0.53700000000000003},"min":{"1min":0.375,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.74299999999999999,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51300000000000001,"15min":0.50800000000000001},"min":{"1min":0.36199999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.70399999999999996,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.65300000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:41:09 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.51200000000000001,"15min":0.51300000000000001},"min":{"1min":0.35199999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.63800000000000001,"5min":0.88,"15min":0.88},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.54400000000000004,"15min":0.54300000000000004},"min":{"1min":0.372,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.71499999999999997,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.70799999999999996}]},{"osd":3,"last update":"Fri Apr 26 04:41:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.53800000000000003,"15min":0.54400000000000004},"min":{"1min":0.42099999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.67500000000000004,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.53500000000000003,"15min":0.52300000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.72999999999999998,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.755}]},{"osd":4,"last update":"Fri Apr 26 04:41:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.438,"15min":0.40799999999999997},"min":{"1min":0.33600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.70099999999999996,"5min":1.103,"15min":1.103},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.443,"15min":0.41499999999999998},"min":{"1min":0.38700000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.63100000000000001,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.73799999999999999}]},{"osd":6,"last update":"Fri Apr 26 04:41:37 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.46899999999999997,"15min":0.45200000000000001},"min":{"1min":0.32000000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.71999999999999997,"5min":1.159,"15min":1.159},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.52000000000000002,"15min":0.5},"min":{"1min":0.40699999999999997,"5min":0.307,"15min":0.307},"max":{"1min":0.65800000000000003,"5min":1.133,"15min":1.133},"last":0.72799999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:41:00 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.52300000000000002,"15min":0.51900000000000002},"min":{"1min":0.36699999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.71599999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.56699999999999995,"15min":0.55900000000000005},"min":{"1min":0.45100000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.76300000000000001,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.67200000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574914,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":688,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":704512,"data_stored":306414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:41:37 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.36599999999999999,"15min":0.34100000000000003},"min":{"1min":0.24199999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.375,"15min":0.35699999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.47999999999999998,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.34200000000000003}]},{"osd":2,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.39000000000000001,"15min":0.38300000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.374},{"interface":"front","average":{"1min":0.372,"5min":0.38400000000000001,"15min":0.378},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.29099999999999998}]},{"osd":3,"last update":"Fri Apr 26 04:41:32 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.40899999999999997,"15min":0.41999999999999998},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.66300000000000003,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.42299999999999999,"15min":0.42899999999999999},"min":{"1min":0.307,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.58699999999999997,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.32700000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:40:52 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56599999999999995,"15min":0.56899999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.751,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.65800000000000003}]},{"osd":5,"last update":"Fri Apr 26 04:41:12 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.61599999999999999,"15min":0.628},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.81000000000000005,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.627},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.61299999999999999,"15min":0.627},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.83699999999999997,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.49199999999999999}]},{"osd":6,"last update":"Fri Apr 26 04:41:37 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.60899999999999999,"15min":0.62},"min":{"1min":0.45800000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.71599999999999997,"5min":0.879,"15min":0.879},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.624,"5min":0.63200000000000001,"15min":0.65000000000000002},"min":{"1min":0.44400000000000001,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.86299999999999999,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.59599999999999997}]},{"osd":7,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.57299999999999995,"15min":0.57499999999999996},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.73099999999999998,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.59399999999999997,"15min":0.59199999999999997},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.63700000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298052,"kb_used_data":696,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447100,"statfs":{"total":95995035648,"available":95689830400,"internally_reserved":0,"allocated":712704,"data_stored":307018,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:41:04 2024","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.375,"15min":0.36499999999999999},"min":{"1min":0.23100000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.49199999999999999,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.33600000000000002,"5min":0.39100000000000001,"15min":0.39200000000000002},"min":{"1min":0.23599999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.55500000000000005,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.41799999999999998}]},{"osd":1,"last update":"Fri Apr 26 04:41:04 2024","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.38800000000000001,"15min":0.38200000000000001},"min":{"1min":0.24399999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.53000000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.41299999999999998,"15min":0.40899999999999997},"min":{"1min":0.27900000000000003,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.54200000000000004,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.36799999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:41:27 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.45000000000000001,"15min":0.46700000000000003},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.58699999999999997,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.46899999999999997,"15min":0.497},"min":{"1min":0.28799999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.51700000000000002,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.44400000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:40:48 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.56399999999999995,"15min":0.56899999999999995},"min":{"1min":0.42899999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59699999999999998,"15min":0.60099999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.83999999999999997,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.78000000000000003}]},{"osd":5,"last update":"Fri Apr 26 04:41:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.57599999999999996,"15min":0.57299999999999995},"min":{"1min":0.442,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.55500000000000005,"15min":0.55000000000000004},"min":{"1min":0.40799999999999997,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.70399999999999996}]},{"osd":6,"last update":"Fri Apr 26 04:41:39 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.58099999999999996,"15min":0.56000000000000005},"min":{"1min":0.41799999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.84799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.58999999999999997,"15min":0.57799999999999996},"min":{"1min":0.46500000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.81000000000000005,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.64700000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:41:04 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.61899999999999999,"15min":0.621},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.872,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.621,"5min":0.62,"15min":0.62},"min":{"1min":0.48699999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.81399999999999995,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.68400000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298632,"kb_used_data":1276,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446520,"statfs":{"total":95995035648,"available":95689236480,"internally_reserved":0,"allocated":1306624,"data_stored":897374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:40:41 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.36099999999999999,"15min":0.34000000000000002},"min":{"1min":0.26300000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.57099999999999995,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.372,"5min":0.38800000000000001,"15min":0.35799999999999998},"min":{"1min":0.27200000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.51400000000000001,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.34000000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:41:03 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.42699999999999999,"15min":0.40500000000000003},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.655,"5min":1.655,"15min":1.655},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.40400000000000003,"15min":0.39500000000000002},"min":{"1min":0.26500000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.42899999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:41:30 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47999999999999998,"15min":0.48999999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.524,"5min":1.524,"15min":1.524},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.44,"5min":0.44400000000000001,"15min":0.45600000000000002},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.3560000000000001,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":0.46000000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:40:48 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.57799999999999996,"15min":0.58499999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.622,"5min":0.60399999999999998,"15min":0.60599999999999998},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.5700000000000001,"5min":1.5700000000000001,"15min":1.5700000000000001},"last":0.51500000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:41:14 2024","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.59699999999999998,"15min":0.59999999999999998},"min":{"1min":0.45800000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.4239999999999999,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.64600000000000002,"15min":0.65500000000000003},"min":{"1min":0.44900000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.599,"5min":1.599,"15min":1.599},"last":0.59499999999999997}]},{"osd":6,"last update":"Fri Apr 26 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.58699999999999997,"15min":0.58299999999999996},"min":{"1min":0.45600000000000002,"5min":0.379,"15min":0.379},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.64800000000000002,"15min":0.65500000000000003},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.6279999999999999,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":0.54600000000000004}]},{"osd":7,"last update":"Fri Apr 26 04:41:03 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.56499999999999995,"15min":0.55500000000000005},"min":{"1min":0.42899999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.3879999999999999,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":0.505},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.59299999999999997,"15min":0.58099999999999996},"min":{"1min":0.46800000000000003,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.52800000000000002}]}]},{"osd":7,"up_from":54,"seq":231928234020,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298648,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446504,"statfs":{"total":95995035648,"available":95689220096,"internally_reserved":0,"allocated":1323008,"data_stored":898227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.51700000000000002,"15min":0.51600000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.53100000000000003,"15min":0.53400000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.71699999999999997,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.57299999999999995}]},{"osd":1,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.54100000000000004,"15min":0.53900000000000003},"min":{"1min":0.32100000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.504,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.504}]},{"osd":2,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.56299999999999994,"15min":0.56899999999999995},"min":{"1min":0.40500000000000003,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.54200000000000004,"15min":0.54600000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.55700000000000005}]},{"osd":3,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55900000000000005,"15min":0.56000000000000005},"min":{"1min":0.42499999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.78800000000000003,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.58099999999999996,"15min":0.58599999999999997},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.76800000000000002,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.65400000000000003}]},{"osd":4,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.52000000000000002,"15min":0.52100000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.1160000000000001,"5min":1.1160000000000001,"15min":1.1160000000000001},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54100000000000004,"15min":0.53900000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.70699999999999996,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.66800000000000004}]},{"osd":5,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.52800000000000002,"15min":0.52600000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.71899999999999997,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.55900000000000005,"15min":0.56100000000000005},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.755,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.61199999999999999}]},{"osd":6,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.57499999999999996,"15min":0.57099999999999995},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":1.046,"5min":1.046,"15min":1.046},"last":0.623},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.754}]}]},{"osd":3,"up_from":25,"seq":107374182456,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298052,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447100,"statfs":{"total":95995035648,"available":95689830400,"internally_reserved":0,"allocated":708608,"data_stored":306504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:41:26 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.373,"15min":0.38100000000000001},"min":{"1min":0.245,"5min":0.219,"15min":0.219},"max":{"1min":0.67900000000000005,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.39200000000000002,"15min":0.40300000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.61499999999999999,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.36899999999999999}]},{"osd":1,"last update":"Fri Apr 26 04:41:26 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.42199999999999999,"15min":0.443},"min":{"1min":0.26100000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.58099999999999996,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.42699999999999999,"15min":0.437},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":0.432}]},{"osd":2,"last update":"Fri Apr 26 04:41:26 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.437,"15min":0.45300000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.2070000000000001,"5min":1.2070000000000001,"15min":1.2070000000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.41399999999999998,"15min":0.41999999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.242,"5min":1.242,"15min":1.242},"last":0.40999999999999998}]},{"osd":4,"last update":"Fri Apr 26 04:40:46 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.42999999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.82999999999999996,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.56000000000000005,"15min":0.55900000000000005},"min":{"1min":0.41699999999999998,"5min":0.373,"15min":0.373},"max":{"1min":0.79000000000000004,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.70599999999999996}]},{"osd":5,"last update":"Fri Apr 26 04:41:08 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56200000000000006,"15min":0.56799999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56699999999999995,"15min":0.56599999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.59999999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:40:39 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.55900000000000005,"15min":0.55100000000000005},"min":{"1min":0.41199999999999998,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.56799999999999995,"15min":0.56200000000000006},"min":{"1min":0.40600000000000003,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.61099999999999999}]},{"osd":7,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.55300000000000005,"15min":0.55100000000000005},"min":{"1min":0.42699999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.71399999999999997,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.57199999999999995,"15min":0.56599999999999995},"min":{"1min":0.441,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.075,"5min":1.075,"15min":1.075},"last":0.59299999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018931,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":729088,"data_stored":308112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:48 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.52500000000000002,"15min":0.51100000000000001},"min":{"1min":0.41099999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.80100000000000005,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.52300000000000002,"15min":0.51500000000000001},"min":{"1min":0.40400000000000003,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.49199999999999999}]},{"osd":1,"last update":"Fri Apr 26 04:40:48 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.33500000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.53200000000000003,"15min":0.51800000000000002},"min":{"1min":0.41899999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.70499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.627}]},{"osd":2,"last update":"Fri Apr 26 04:40:48 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.54900000000000004,"15min":0.54100000000000004},"min":{"1min":0.39800000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.753,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.54300000000000004,"15min":0.54100000000000004},"min":{"1min":0.435,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.78900000000000003,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.60399999999999998}]},{"osd":3,"last update":"Fri Apr 26 04:40:48 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.45100000000000001,"5min":0.372,"15min":0.372},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.53100000000000003,"15min":0.52500000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.57499999999999996}]},{"osd":5,"last update":"Fri Apr 26 04:41:10 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.46300000000000002,"15min":0.45500000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.72599999999999998,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.497,"5min":0.46800000000000003,"15min":0.45000000000000001},"min":{"1min":0.35099999999999998,"5min":0.318,"15min":0.318},"max":{"1min":0.65900000000000003,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.55300000000000005}]},{"osd":6,"last update":"Fri Apr 26 04:41:35 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.5,"15min":0.48799999999999999},"min":{"1min":0.314,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.68500000000000005,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.53400000000000003,"15min":0.53000000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.75700000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.74399999999999999}]},{"osd":7,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56699999999999995,"15min":0.56799999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.53200000000000003,"15min":0.53600000000000003},"min":{"1min":0.38200000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.61699999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495657,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298640,"kb_used_data":1280,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446512,"statfs":{"total":95995035648,"available":95689228288,"internally_reserved":0,"allocated":1310720,"data_stored":898131,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.33900000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.67700000000000005,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.51400000000000001,"15min":0.52600000000000002},"min":{"1min":0.36799999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.58199999999999996,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.46100000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52600000000000002,"15min":0.52900000000000003},"min":{"1min":0.39600000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.64300000000000002,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.55700000000000005,"15min":0.57499999999999996},"min":{"1min":0.36099999999999999,"5min":0.317,"15min":0.317},"max":{"1min":0.62,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.58099999999999996}]},{"osd":2,"last update":"Fri Apr 26 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.53700000000000003,"15min":0.54200000000000004},"min":{"1min":0.39800000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.64300000000000002,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.54900000000000004,"15min":0.55200000000000005},"min":{"1min":0.45600000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.67900000000000005,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.56000000000000005}]},{"osd":3,"last update":"Fri Apr 26 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.56999999999999995,"15min":0.57899999999999996},"min":{"1min":0.47699999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.69299999999999995,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.505,"5min":0.52800000000000002,"15min":0.53400000000000003},"min":{"1min":0.39700000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.627,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.59899999999999998}]},{"osd":4,"last update":"Fri Apr 26 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.437,"15min":0.42999999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.504,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.504},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.47999999999999998,"15min":0.47499999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.66500000000000004,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.40200000000000002}]},{"osd":5,"last update":"Fri Apr 26 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.45800000000000002,"15min":0.45200000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.65300000000000002,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51400000000000001,"15min":0.503},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.71199999999999997,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.67500000000000004}]},{"osd":7,"last update":"Fri Apr 26 04:40:59 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.34000000000000002,"5min":0.312,"15min":0.312},"max":{"1min":0.73699999999999999,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.56499999999999995,"15min":0.56399999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.75800000000000001,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.65100000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:41:44.770 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:41:45.023 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:45.447 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:45 smithi156 ceph-mon[27570]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:41:45.447 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:45 smithi156 ceph-mon[39130]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:41:45.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:45 smithi171 ceph-mon[31139]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:41:46.586 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:41:46 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:41:46] "GET /metrics HTTP/1.1" 200 239117 "" "Prometheus/2.43.0" 2024-04-26T04:41:46.586 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:46 smithi156 ceph-mon[27570]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:46.586 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:46 smithi156 ceph-mon[39130]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:46.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:46 smithi171 ceph-mon[31139]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:47.040 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:41:47.042 INFO:teuthology.orchestra.run.smithi156.stderr:dumped all 2024-04-26T04:41:47.825 INFO:teuthology.orchestra.run.smithi156.stdout:{"pg_ready":true,"pg_map":{"version":80,"stamp":"2024-04-26T04:41:45.219723+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386200,"kb_used_data":7332,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575016,"statfs":{"total":767960285184,"available":765516816384,"internally_reserved":0,"allocated":7507968,"data_stored":4228834,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002613"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.297568+0000","last_change":"2024-04-26T04:40:44.295432+0000","last_active":"2024-04-26T04:41:29.297568+0000","last_peered":"2024-04-26T04:41:29.297568+0000","last_clean":"2024-04-26T04:41:29.297568+0000","last_became_active":"2024-04-26T04:40:44.295146+0000","last_became_peered":"2024-04-26T04:40:44.295146+0000","last_unstale":"2024-04-26T04:41:29.297568+0000","last_undegraded":"2024-04-26T04:41:29.297568+0000","last_fullsized":"2024-04-26T04:41:29.297568+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:53:42.927961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457134+0000","last_change":"2024-04-26T04:40:44.269868+0000","last_active":"2024-04-26T04:41:29.457134+0000","last_peered":"2024-04-26T04:41:29.457134+0000","last_clean":"2024-04-26T04:41:29.457134+0000","last_became_active":"2024-04-26T04:40:44.269581+0000","last_became_peered":"2024-04-26T04:40:44.269581+0000","last_unstale":"2024-04-26T04:41:29.457134+0000","last_undegraded":"2024-04-26T04:41:29.457134+0000","last_fullsized":"2024-04-26T04:41:29.457134+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:57:53.521833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.293926+0000","last_change":"2024-04-26T04:40:44.293680+0000","last_active":"2024-04-26T04:41:29.293926+0000","last_peered":"2024-04-26T04:41:29.293926+0000","last_clean":"2024-04-26T04:41:29.293926+0000","last_became_active":"2024-04-26T04:40:44.293427+0000","last_became_peered":"2024-04-26T04:40:44.293427+0000","last_unstale":"2024-04-26T04:41:29.293926+0000","last_undegraded":"2024-04-26T04:41:29.293926+0000","last_fullsized":"2024-04-26T04:41:29.293926+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:29:58.842791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294458+0000","last_change":"2024-04-26T04:40:44.266052+0000","last_active":"2024-04-26T04:41:29.294458+0000","last_peered":"2024-04-26T04:41:29.294458+0000","last_clean":"2024-04-26T04:41:29.294458+0000","last_became_active":"2024-04-26T04:40:44.265778+0000","last_became_peered":"2024-04-26T04:40:44.265778+0000","last_unstale":"2024-04-26T04:41:29.294458+0000","last_undegraded":"2024-04-26T04:41:29.294458+0000","last_fullsized":"2024-04-26T04:41:29.294458+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:20:20.459628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.298607+0000","last_change":"2024-04-26T04:40:44.296114+0000","last_active":"2024-04-26T04:41:29.298607+0000","last_peered":"2024-04-26T04:41:29.298607+0000","last_clean":"2024-04-26T04:41:29.298607+0000","last_became_active":"2024-04-26T04:40:44.295707+0000","last_became_peered":"2024-04-26T04:40:44.295707+0000","last_unstale":"2024-04-26T04:41:29.298607+0000","last_undegraded":"2024-04-26T04:41:29.298607+0000","last_fullsized":"2024-04-26T04:41:29.298607+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:59:16.393307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294403+0000","last_change":"2024-04-26T04:40:44.269169+0000","last_active":"2024-04-26T04:41:29.294403+0000","last_peered":"2024-04-26T04:41:29.294403+0000","last_clean":"2024-04-26T04:41:29.294403+0000","last_became_active":"2024-04-26T04:40:44.268903+0000","last_became_peered":"2024-04-26T04:40:44.268903+0000","last_unstale":"2024-04-26T04:41:29.294403+0000","last_undegraded":"2024-04-26T04:41:29.294403+0000","last_fullsized":"2024-04-26T04:41:29.294403+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:07:22.671960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.298044+0000","last_change":"2024-04-26T04:40:44.295815+0000","last_active":"2024-04-26T04:41:29.298044+0000","last_peered":"2024-04-26T04:41:29.298044+0000","last_clean":"2024-04-26T04:41:29.298044+0000","last_became_active":"2024-04-26T04:40:44.295501+0000","last_became_peered":"2024-04-26T04:40:44.295501+0000","last_unstale":"2024-04-26T04:41:29.298044+0000","last_undegraded":"2024-04-26T04:41:29.298044+0000","last_fullsized":"2024-04-26T04:41:29.298044+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:21:41.810593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294268+0000","last_change":"2024-04-26T04:40:44.270149+0000","last_active":"2024-04-26T04:41:29.294268+0000","last_peered":"2024-04-26T04:41:29.294268+0000","last_clean":"2024-04-26T04:41:29.294268+0000","last_became_active":"2024-04-26T04:40:44.269847+0000","last_became_peered":"2024-04-26T04:40:44.269847+0000","last_unstale":"2024-04-26T04:41:29.294268+0000","last_undegraded":"2024-04-26T04:41:29.294268+0000","last_fullsized":"2024-04-26T04:41:29.294268+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:28:48.656498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.295874+0000","last_change":"2024-04-26T04:40:44.294836+0000","last_active":"2024-04-26T04:41:29.295874+0000","last_peered":"2024-04-26T04:41:29.295874+0000","last_clean":"2024-04-26T04:41:29.295874+0000","last_became_active":"2024-04-26T04:40:44.294525+0000","last_became_peered":"2024-04-26T04:40:44.294525+0000","last_unstale":"2024-04-26T04:41:29.295874+0000","last_undegraded":"2024-04-26T04:41:29.295874+0000","last_fullsized":"2024-04-26T04:41:29.295874+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:48:08.301019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.456975+0000","last_change":"2024-04-26T04:40:44.272691+0000","last_active":"2024-04-26T04:41:29.456975+0000","last_peered":"2024-04-26T04:41:29.456975+0000","last_clean":"2024-04-26T04:41:29.456975+0000","last_became_active":"2024-04-26T04:40:44.272339+0000","last_became_peered":"2024-04-26T04:40:44.272339+0000","last_unstale":"2024-04-26T04:41:29.456975+0000","last_undegraded":"2024-04-26T04:41:29.456975+0000","last_fullsized":"2024-04-26T04:41:29.456975+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:54:52.047487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.297926+0000","last_change":"2024-04-26T04:40:44.288207+0000","last_active":"2024-04-26T04:41:29.297926+0000","last_peered":"2024-04-26T04:41:29.297926+0000","last_clean":"2024-04-26T04:41:29.297926+0000","last_became_active":"2024-04-26T04:40:44.287939+0000","last_became_peered":"2024-04-26T04:40:44.287939+0000","last_unstale":"2024-04-26T04:41:29.297926+0000","last_undegraded":"2024-04-26T04:41:29.297926+0000","last_fullsized":"2024-04-26T04:41:29.297926+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:20:35.276921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457064+0000","last_change":"2024-04-26T04:40:44.275233+0000","last_active":"2024-04-26T04:41:29.457064+0000","last_peered":"2024-04-26T04:41:29.457064+0000","last_clean":"2024-04-26T04:41:29.457064+0000","last_became_active":"2024-04-26T04:40:44.274961+0000","last_became_peered":"2024-04-26T04:40:44.274961+0000","last_unstale":"2024-04-26T04:41:29.457064+0000","last_undegraded":"2024-04-26T04:41:29.457064+0000","last_fullsized":"2024-04-26T04:41:29.457064+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:10:05.950012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.300489+0000","last_change":"2024-04-26T04:39:29.193470+0000","last_active":"2024-04-26T04:41:29.300489+0000","last_peered":"2024-04-26T04:41:29.300489+0000","last_clean":"2024-04-26T04:41:29.300489+0000","last_became_active":"2024-04-26T04:39:29.193156+0000","last_became_peered":"2024-04-26T04:39:29.193156+0000","last_unstale":"2024-04-26T04:41:29.300489+0000","last_undegraded":"2024-04-26T04:41:29.300489+0000","last_fullsized":"2024-04-26T04:41:29.300489+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:16:39.309651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294192+0000","last_change":"2024-04-26T04:40:44.865475+0000","last_active":"2024-04-26T04:41:29.294192+0000","last_peered":"2024-04-26T04:41:29.294192+0000","last_clean":"2024-04-26T04:41:29.294192+0000","last_became_active":"2024-04-26T04:40:44.865022+0000","last_became_peered":"2024-04-26T04:40:44.865022+0000","last_unstale":"2024-04-26T04:41:29.294192+0000","last_undegraded":"2024-04-26T04:41:29.294192+0000","last_fullsized":"2024-04-26T04:41:29.294192+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:37:57.242315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.295727+0000","last_change":"2024-04-26T04:39:29.193876+0000","last_active":"2024-04-26T04:41:29.295727+0000","last_peered":"2024-04-26T04:41:29.295727+0000","last_clean":"2024-04-26T04:41:29.295727+0000","last_became_active":"2024-04-26T04:39:29.193532+0000","last_became_peered":"2024-04-26T04:39:29.193532+0000","last_unstale":"2024-04-26T04:41:29.295727+0000","last_undegraded":"2024-04-26T04:41:29.295727+0000","last_fullsized":"2024-04-26T04:41:29.295727+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:58:26.728897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.295311+0000","last_change":"2024-04-26T04:40:44.269328+0000","last_active":"2024-04-26T04:41:29.295311+0000","last_peered":"2024-04-26T04:41:29.295311+0000","last_clean":"2024-04-26T04:41:29.295311+0000","last_became_active":"2024-04-26T04:40:44.269047+0000","last_became_peered":"2024-04-26T04:40:44.269047+0000","last_unstale":"2024-04-26T04:41:29.295311+0000","last_undegraded":"2024-04-26T04:41:29.295311+0000","last_fullsized":"2024-04-26T04:41:29.295311+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:30:56.122947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.456829+0000","last_change":"2024-04-26T04:40:44.865161+0000","last_active":"2024-04-26T04:41:29.456829+0000","last_peered":"2024-04-26T04:41:29.456829+0000","last_clean":"2024-04-26T04:41:29.456829+0000","last_became_active":"2024-04-26T04:40:44.864770+0000","last_became_peered":"2024-04-26T04:40:44.864770+0000","last_unstale":"2024-04-26T04:41:29.456829+0000","last_undegraded":"2024-04-26T04:41:29.456829+0000","last_fullsized":"2024-04-26T04:41:29.456829+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:10:58.140636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.295554+0000","last_change":"2024-04-26T04:40:44.264938+0000","last_active":"2024-04-26T04:41:29.295554+0000","last_peered":"2024-04-26T04:41:29.295554+0000","last_clean":"2024-04-26T04:41:29.295554+0000","last_became_active":"2024-04-26T04:40:44.264670+0000","last_became_peered":"2024-04-26T04:40:44.264670+0000","last_unstale":"2024-04-26T04:41:29.295554+0000","last_undegraded":"2024-04-26T04:41:29.295554+0000","last_fullsized":"2024-04-26T04:41:29.295554+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:21:05.870793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457679+0000","last_change":"2024-04-26T04:39:29.224943+0000","last_active":"2024-04-26T04:41:29.457679+0000","last_peered":"2024-04-26T04:41:29.457679+0000","last_clean":"2024-04-26T04:41:29.457679+0000","last_became_active":"2024-04-26T04:39:29.224586+0000","last_became_peered":"2024-04-26T04:39:29.224586+0000","last_unstale":"2024-04-26T04:41:29.457679+0000","last_undegraded":"2024-04-26T04:41:29.457679+0000","last_fullsized":"2024-04-26T04:41:29.457679+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:50:41.653796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457113+0000","last_change":"2024-04-26T04:40:44.293195+0000","last_active":"2024-04-26T04:41:29.457113+0000","last_peered":"2024-04-26T04:41:29.457113+0000","last_clean":"2024-04-26T04:41:29.457113+0000","last_became_active":"2024-04-26T04:40:44.292921+0000","last_became_peered":"2024-04-26T04:40:44.292921+0000","last_unstale":"2024-04-26T04:41:29.457113+0000","last_undegraded":"2024-04-26T04:41:29.457113+0000","last_fullsized":"2024-04-26T04:41:29.457113+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:31:15.459441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.456857+0000","last_change":"2024-04-26T04:40:44.270682+0000","last_active":"2024-04-26T04:41:29.456857+0000","last_peered":"2024-04-26T04:41:29.456857+0000","last_clean":"2024-04-26T04:41:29.456857+0000","last_became_active":"2024-04-26T04:40:44.270404+0000","last_became_peered":"2024-04-26T04:40:44.270404+0000","last_unstale":"2024-04-26T04:41:29.456857+0000","last_undegraded":"2024-04-26T04:41:29.456857+0000","last_fullsized":"2024-04-26T04:41:29.456857+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:48:44.848733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457413+0000","last_change":"2024-04-26T04:39:29.224807+0000","last_active":"2024-04-26T04:41:29.457413+0000","last_peered":"2024-04-26T04:41:29.457413+0000","last_clean":"2024-04-26T04:41:29.457413+0000","last_became_active":"2024-04-26T04:39:29.224497+0000","last_became_peered":"2024-04-26T04:39:29.224497+0000","last_unstale":"2024-04-26T04:41:29.457413+0000","last_undegraded":"2024-04-26T04:41:29.457413+0000","last_fullsized":"2024-04-26T04:41:29.457413+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:20:03.848049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.300470+0000","last_change":"2024-04-26T04:40:44.864165+0000","last_active":"2024-04-26T04:41:29.300470+0000","last_peered":"2024-04-26T04:41:29.300470+0000","last_clean":"2024-04-26T04:41:29.300470+0000","last_became_active":"2024-04-26T04:40:44.863874+0000","last_became_peered":"2024-04-26T04:40:44.863874+0000","last_unstale":"2024-04-26T04:41:29.300470+0000","last_undegraded":"2024-04-26T04:41:29.300470+0000","last_fullsized":"2024-04-26T04:41:29.300470+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:32:35.821759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.293847+0000","last_change":"2024-04-26T04:40:44.265221+0000","last_active":"2024-04-26T04:41:29.293847+0000","last_peered":"2024-04-26T04:41:29.293847+0000","last_clean":"2024-04-26T04:41:29.293847+0000","last_became_active":"2024-04-26T04:40:44.264934+0000","last_became_peered":"2024-04-26T04:40:44.264934+0000","last_unstale":"2024-04-26T04:41:29.293847+0000","last_undegraded":"2024-04-26T04:41:29.293847+0000","last_fullsized":"2024-04-26T04:41:29.293847+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:56:43.564670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"62'128","reported_seq":276,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457373+0000","last_change":"2024-04-26T04:39:00.628070+0000","last_active":"2024-04-26T04:41:29.457373+0000","last_peered":"2024-04-26T04:41:29.457373+0000","last_clean":"2024-04-26T04:41:29.457373+0000","last_became_active":"2024-04-26T04:39:00.627546+0000","last_became_peered":"2024-04-26T04:39:00.627546+0000","last_unstale":"2024-04-26T04:41:29.457373+0000","last_undegraded":"2024-04-26T04:41:29.457373+0000","last_fullsized":"2024-04-26T04:41:29.457373+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:36:55.213599+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:36:55.213599+0000","last_clean_scrub_stamp":"2024-04-26T04:36:55.213599+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:26:44.288000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294121+0000","last_change":"2024-04-26T04:39:30.840097+0000","last_active":"2024-04-26T04:41:29.294121+0000","last_peered":"2024-04-26T04:41:29.294121+0000","last_clean":"2024-04-26T04:41:29.294121+0000","last_became_active":"2024-04-26T04:39:30.839750+0000","last_became_peered":"2024-04-26T04:39:30.839750+0000","last_unstale":"2024-04-26T04:41:29.294121+0000","last_undegraded":"2024-04-26T04:41:29.294121+0000","last_fullsized":"2024-04-26T04:41:29.294121+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:00:14.008690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294850+0000","last_change":"2024-04-26T04:40:44.299462+0000","last_active":"2024-04-26T04:41:29.294850+0000","last_peered":"2024-04-26T04:41:29.294850+0000","last_clean":"2024-04-26T04:41:29.294850+0000","last_became_active":"2024-04-26T04:40:44.298852+0000","last_became_peered":"2024-04-26T04:40:44.298852+0000","last_unstale":"2024-04-26T04:41:29.294850+0000","last_undegraded":"2024-04-26T04:41:29.294850+0000","last_fullsized":"2024-04-26T04:41:29.294850+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:37:55.693467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294042+0000","last_change":"2024-04-26T04:40:44.265714+0000","last_active":"2024-04-26T04:41:29.294042+0000","last_peered":"2024-04-26T04:41:29.294042+0000","last_clean":"2024-04-26T04:41:29.294042+0000","last_became_active":"2024-04-26T04:40:44.265299+0000","last_became_peered":"2024-04-26T04:40:44.265299+0000","last_unstale":"2024-04-26T04:41:29.294042+0000","last_undegraded":"2024-04-26T04:41:29.294042+0000","last_fullsized":"2024-04-26T04:41:29.294042+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:49:33.627495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.296137+0000","last_change":"2024-04-26T04:39:29.224560+0000","last_active":"2024-04-26T04:41:29.296137+0000","last_peered":"2024-04-26T04:41:29.296137+0000","last_clean":"2024-04-26T04:41:29.296137+0000","last_became_active":"2024-04-26T04:39:29.224239+0000","last_became_peered":"2024-04-26T04:39:29.224239+0000","last_unstale":"2024-04-26T04:41:29.296137+0000","last_undegraded":"2024-04-26T04:41:29.296137+0000","last_fullsized":"2024-04-26T04:41:29.296137+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:27:28.097185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.295488+0000","last_change":"2024-04-26T04:40:44.297277+0000","last_active":"2024-04-26T04:41:29.295488+0000","last_peered":"2024-04-26T04:41:29.295488+0000","last_clean":"2024-04-26T04:41:29.295488+0000","last_became_active":"2024-04-26T04:40:44.295885+0000","last_became_peered":"2024-04-26T04:40:44.295885+0000","last_unstale":"2024-04-26T04:41:29.295488+0000","last_undegraded":"2024-04-26T04:41:29.295488+0000","last_fullsized":"2024-04-26T04:41:29.295488+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:07:25.143389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.297839+0000","last_change":"2024-04-26T04:40:44.271779+0000","last_active":"2024-04-26T04:41:29.297839+0000","last_peered":"2024-04-26T04:41:29.297839+0000","last_clean":"2024-04-26T04:41:29.297839+0000","last_became_active":"2024-04-26T04:40:44.270674+0000","last_became_peered":"2024-04-26T04:40:44.270674+0000","last_unstale":"2024-04-26T04:41:29.297839+0000","last_undegraded":"2024-04-26T04:41:29.297839+0000","last_fullsized":"2024-04-26T04:41:29.297839+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:37:12.036437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294007+0000","last_change":"2024-04-26T04:39:30.545245+0000","last_active":"2024-04-26T04:41:29.294007+0000","last_peered":"2024-04-26T04:41:29.294007+0000","last_clean":"2024-04-26T04:41:29.294007+0000","last_became_active":"2024-04-26T04:39:30.544940+0000","last_became_peered":"2024-04-26T04:39:30.544940+0000","last_unstale":"2024-04-26T04:41:29.294007+0000","last_undegraded":"2024-04-26T04:41:29.294007+0000","last_fullsized":"2024-04-26T04:41:29.294007+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:53:32.749890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:41:30.299248+0000","last_change":"2024-04-26T04:41:30.299248+0000","last_active":"2024-04-26T04:41:30.299248+0000","last_peered":"2024-04-26T04:41:30.299248+0000","last_clean":"2024-04-26T04:41:30.299248+0000","last_became_active":"2024-04-26T04:41:30.299009+0000","last_became_peered":"2024-04-26T04:41:30.299009+0000","last_unstale":"2024-04-26T04:41:30.299248+0000","last_undegraded":"2024-04-26T04:41:30.299248+0000","last_fullsized":"2024-04-26T04:41:30.299248+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:29:16.315075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294416+0000","last_change":"2024-04-26T04:40:44.265375+0000","last_active":"2024-04-26T04:41:29.294416+0000","last_peered":"2024-04-26T04:41:29.294416+0000","last_clean":"2024-04-26T04:41:29.294416+0000","last_became_active":"2024-04-26T04:40:44.265077+0000","last_became_peered":"2024-04-26T04:40:44.265077+0000","last_unstale":"2024-04-26T04:41:29.294416+0000","last_undegraded":"2024-04-26T04:41:29.294416+0000","last_fullsized":"2024-04-26T04:41:29.294416+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:37:54.273375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.295422+0000","last_change":"2024-04-26T04:39:29.224546+0000","last_active":"2024-04-26T04:41:29.295422+0000","last_peered":"2024-04-26T04:41:29.295422+0000","last_clean":"2024-04-26T04:41:29.295422+0000","last_became_active":"2024-04-26T04:39:29.224238+0000","last_became_peered":"2024-04-26T04:39:29.224238+0000","last_unstale":"2024-04-26T04:41:29.295422+0000","last_undegraded":"2024-04-26T04:41:29.295422+0000","last_fullsized":"2024-04-26T04:41:29.295422+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:45:24.471564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:41:30.301211+0000","last_change":"2024-04-26T04:41:30.301211+0000","last_active":"2024-04-26T04:41:30.301211+0000","last_peered":"2024-04-26T04:41:30.301211+0000","last_clean":"2024-04-26T04:41:30.301211+0000","last_became_active":"2024-04-26T04:41:30.300881+0000","last_became_peered":"2024-04-26T04:41:30.300881+0000","last_unstale":"2024-04-26T04:41:30.301211+0000","last_undegraded":"2024-04-26T04:41:30.301211+0000","last_fullsized":"2024-04-26T04:41:30.301211+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:28:09.438146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.293485+0000","last_change":"2024-04-26T04:40:44.263300+0000","last_active":"2024-04-26T04:41:29.293485+0000","last_peered":"2024-04-26T04:41:29.293485+0000","last_clean":"2024-04-26T04:41:29.293485+0000","last_became_active":"2024-04-26T04:40:44.263005+0000","last_became_peered":"2024-04-26T04:40:44.263005+0000","last_unstale":"2024-04-26T04:41:29.293485+0000","last_undegraded":"2024-04-26T04:41:29.293485+0000","last_fullsized":"2024-04-26T04:41:29.293485+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:45:56.407034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":22,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:41:30.559476+0000","last_change":"2024-04-26T04:41:30.559476+0000","last_active":"2024-04-26T04:41:30.559476+0000","last_peered":"2024-04-26T04:41:30.559476+0000","last_clean":"2024-04-26T04:41:30.559476+0000","last_became_active":"2024-04-26T04:41:30.559052+0000","last_became_peered":"2024-04-26T04:41:30.559052+0000","last_unstale":"2024-04-26T04:41:30.559476+0000","last_undegraded":"2024-04-26T04:41:30.559476+0000","last_fullsized":"2024-04-26T04:41:30.559476+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:56:25.173257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.297964+0000","last_change":"2024-04-26T04:40:44.270029+0000","last_active":"2024-04-26T04:41:29.297964+0000","last_peered":"2024-04-26T04:41:29.297964+0000","last_clean":"2024-04-26T04:41:29.297964+0000","last_became_active":"2024-04-26T04:40:44.268568+0000","last_became_peered":"2024-04-26T04:40:44.268568+0000","last_unstale":"2024-04-26T04:41:29.297964+0000","last_undegraded":"2024-04-26T04:41:29.297964+0000","last_fullsized":"2024-04-26T04:41:29.297964+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:39:19.144986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.298579+0000","last_change":"2024-04-26T04:40:44.294654+0000","last_active":"2024-04-26T04:41:29.298579+0000","last_peered":"2024-04-26T04:41:29.298579+0000","last_clean":"2024-04-26T04:41:29.298579+0000","last_became_active":"2024-04-26T04:40:44.294362+0000","last_became_peered":"2024-04-26T04:40:44.294362+0000","last_unstale":"2024-04-26T04:41:29.298579+0000","last_undegraded":"2024-04-26T04:41:29.298579+0000","last_fullsized":"2024-04-26T04:41:29.298579+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:20:19.295674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.300429+0000","last_change":"2024-04-26T04:40:44.274083+0000","last_active":"2024-04-26T04:41:29.300429+0000","last_peered":"2024-04-26T04:41:29.300429+0000","last_clean":"2024-04-26T04:41:29.300429+0000","last_became_active":"2024-04-26T04:40:44.273811+0000","last_became_peered":"2024-04-26T04:40:44.273811+0000","last_unstale":"2024-04-26T04:41:29.300429+0000","last_undegraded":"2024-04-26T04:41:29.300429+0000","last_fullsized":"2024-04-26T04:41:29.300429+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:33:49.201206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.298151+0000","last_change":"2024-04-26T04:40:44.296732+0000","last_active":"2024-04-26T04:41:29.298151+0000","last_peered":"2024-04-26T04:41:29.298151+0000","last_clean":"2024-04-26T04:41:29.298151+0000","last_became_active":"2024-04-26T04:40:44.296414+0000","last_became_peered":"2024-04-26T04:40:44.296414+0000","last_unstale":"2024-04-26T04:41:29.298151+0000","last_undegraded":"2024-04-26T04:41:29.298151+0000","last_fullsized":"2024-04-26T04:41:29.298151+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:55:50.262968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294052+0000","last_change":"2024-04-26T04:40:44.274301+0000","last_active":"2024-04-26T04:41:29.294052+0000","last_peered":"2024-04-26T04:41:29.294052+0000","last_clean":"2024-04-26T04:41:29.294052+0000","last_became_active":"2024-04-26T04:40:44.273989+0000","last_became_peered":"2024-04-26T04:40:44.273989+0000","last_unstale":"2024-04-26T04:41:29.294052+0000","last_undegraded":"2024-04-26T04:41:29.294052+0000","last_fullsized":"2024-04-26T04:41:29.294052+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:09:09.093513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.456974+0000","last_change":"2024-04-26T04:40:44.292554+0000","last_active":"2024-04-26T04:41:29.456974+0000","last_peered":"2024-04-26T04:41:29.456974+0000","last_clean":"2024-04-26T04:41:29.456974+0000","last_became_active":"2024-04-26T04:40:44.292257+0000","last_became_peered":"2024-04-26T04:40:44.292257+0000","last_unstale":"2024-04-26T04:41:29.456974+0000","last_undegraded":"2024-04-26T04:41:29.456974+0000","last_fullsized":"2024-04-26T04:41:29.456974+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:28:16.184538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294147+0000","last_change":"2024-04-26T04:40:44.267065+0000","last_active":"2024-04-26T04:41:29.294147+0000","last_peered":"2024-04-26T04:41:29.294147+0000","last_clean":"2024-04-26T04:41:29.294147+0000","last_became_active":"2024-04-26T04:40:44.266787+0000","last_became_peered":"2024-04-26T04:40:44.266787+0000","last_unstale":"2024-04-26T04:41:29.294147+0000","last_undegraded":"2024-04-26T04:41:29.294147+0000","last_fullsized":"2024-04-26T04:41:29.294147+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:42:02.342804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.300326+0000","last_change":"2024-04-26T04:40:44.864885+0000","last_active":"2024-04-26T04:41:29.300326+0000","last_peered":"2024-04-26T04:41:29.300326+0000","last_clean":"2024-04-26T04:41:29.300326+0000","last_became_active":"2024-04-26T04:40:44.864612+0000","last_became_peered":"2024-04-26T04:40:44.864612+0000","last_unstale":"2024-04-26T04:41:29.300326+0000","last_undegraded":"2024-04-26T04:41:29.300326+0000","last_fullsized":"2024-04-26T04:41:29.300326+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:31:17.764915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457789+0000","last_change":"2024-04-26T04:40:44.265256+0000","last_active":"2024-04-26T04:41:29.457789+0000","last_peered":"2024-04-26T04:41:29.457789+0000","last_clean":"2024-04-26T04:41:29.457789+0000","last_became_active":"2024-04-26T04:40:44.264777+0000","last_became_peered":"2024-04-26T04:40:44.264777+0000","last_unstale":"2024-04-26T04:41:29.457789+0000","last_undegraded":"2024-04-26T04:41:29.457789+0000","last_fullsized":"2024-04-26T04:41:29.457789+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:05:04.374938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.297761+0000","last_change":"2024-04-26T04:40:44.296722+0000","last_active":"2024-04-26T04:41:29.297761+0000","last_peered":"2024-04-26T04:41:29.297761+0000","last_clean":"2024-04-26T04:41:29.297761+0000","last_became_active":"2024-04-26T04:40:44.296258+0000","last_became_peered":"2024-04-26T04:40:44.296258+0000","last_unstale":"2024-04-26T04:41:29.297761+0000","last_undegraded":"2024-04-26T04:41:29.297761+0000","last_fullsized":"2024-04-26T04:41:29.297761+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:18:01.299512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457803+0000","last_change":"2024-04-26T04:40:44.274325+0000","last_active":"2024-04-26T04:41:29.457803+0000","last_peered":"2024-04-26T04:41:29.457803+0000","last_clean":"2024-04-26T04:41:29.457803+0000","last_became_active":"2024-04-26T04:40:44.274029+0000","last_became_peered":"2024-04-26T04:40:44.274029+0000","last_unstale":"2024-04-26T04:41:29.457803+0000","last_undegraded":"2024-04-26T04:41:29.457803+0000","last_fullsized":"2024-04-26T04:41:29.457803+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:02:00.292982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.293996+0000","last_change":"2024-04-26T04:40:44.864902+0000","last_active":"2024-04-26T04:41:29.293996+0000","last_peered":"2024-04-26T04:41:29.293996+0000","last_clean":"2024-04-26T04:41:29.293996+0000","last_became_active":"2024-04-26T04:40:44.864603+0000","last_became_peered":"2024-04-26T04:40:44.864603+0000","last_unstale":"2024-04-26T04:41:29.293996+0000","last_undegraded":"2024-04-26T04:41:29.293996+0000","last_fullsized":"2024-04-26T04:41:29.293996+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:50:44.169231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457856+0000","last_change":"2024-04-26T04:40:44.269258+0000","last_active":"2024-04-26T04:41:29.457856+0000","last_peered":"2024-04-26T04:41:29.457856+0000","last_clean":"2024-04-26T04:41:29.457856+0000","last_became_active":"2024-04-26T04:40:44.268931+0000","last_became_peered":"2024-04-26T04:40:44.268931+0000","last_unstale":"2024-04-26T04:41:29.457856+0000","last_undegraded":"2024-04-26T04:41:29.457856+0000","last_fullsized":"2024-04-26T04:41:29.457856+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:51:21.554999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.295395+0000","last_change":"2024-04-26T04:40:44.864375+0000","last_active":"2024-04-26T04:41:29.295395+0000","last_peered":"2024-04-26T04:41:29.295395+0000","last_clean":"2024-04-26T04:41:29.295395+0000","last_became_active":"2024-04-26T04:40:44.864045+0000","last_became_peered":"2024-04-26T04:40:44.864045+0000","last_unstale":"2024-04-26T04:41:29.295395+0000","last_undegraded":"2024-04-26T04:41:29.295395+0000","last_fullsized":"2024-04-26T04:41:29.295395+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:44:41.278510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":21,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.300237+0000","last_change":"2024-04-26T04:40:44.272426+0000","last_active":"2024-04-26T04:41:29.300237+0000","last_peered":"2024-04-26T04:41:29.300237+0000","last_clean":"2024-04-26T04:41:29.300237+0000","last_became_active":"2024-04-26T04:40:44.272127+0000","last_became_peered":"2024-04-26T04:40:44.272127+0000","last_unstale":"2024-04-26T04:41:29.300237+0000","last_undegraded":"2024-04-26T04:41:29.300237+0000","last_fullsized":"2024-04-26T04:41:29.300237+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:16:45.181003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.456952+0000","last_change":"2024-04-26T04:40:44.865102+0000","last_active":"2024-04-26T04:41:29.456952+0000","last_peered":"2024-04-26T04:41:29.456952+0000","last_clean":"2024-04-26T04:41:29.456952+0000","last_became_active":"2024-04-26T04:40:44.864785+0000","last_became_peered":"2024-04-26T04:40:44.864785+0000","last_unstale":"2024-04-26T04:41:29.456952+0000","last_undegraded":"2024-04-26T04:41:29.456952+0000","last_fullsized":"2024-04-26T04:41:29.456952+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:12:42.433794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457452+0000","last_change":"2024-04-26T04:40:44.269467+0000","last_active":"2024-04-26T04:41:29.457452+0000","last_peered":"2024-04-26T04:41:29.457452+0000","last_clean":"2024-04-26T04:41:29.457452+0000","last_became_active":"2024-04-26T04:40:44.269173+0000","last_became_peered":"2024-04-26T04:40:44.269173+0000","last_unstale":"2024-04-26T04:41:29.457452+0000","last_undegraded":"2024-04-26T04:41:29.457452+0000","last_fullsized":"2024-04-26T04:41:29.457452+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:17:31.868820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294288+0000","last_change":"2024-04-26T04:40:44.864213+0000","last_active":"2024-04-26T04:41:29.294288+0000","last_peered":"2024-04-26T04:41:29.294288+0000","last_clean":"2024-04-26T04:41:29.294288+0000","last_became_active":"2024-04-26T04:40:44.863912+0000","last_became_peered":"2024-04-26T04:40:44.863912+0000","last_unstale":"2024-04-26T04:41:29.294288+0000","last_undegraded":"2024-04-26T04:41:29.294288+0000","last_fullsized":"2024-04-26T04:41:29.294288+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:38:46.486109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.456901+0000","last_change":"2024-04-26T04:40:44.271893+0000","last_active":"2024-04-26T04:41:29.456901+0000","last_peered":"2024-04-26T04:41:29.456901+0000","last_clean":"2024-04-26T04:41:29.456901+0000","last_became_active":"2024-04-26T04:40:44.271613+0000","last_became_peered":"2024-04-26T04:40:44.271613+0000","last_unstale":"2024-04-26T04:41:29.456901+0000","last_undegraded":"2024-04-26T04:41:29.456901+0000","last_fullsized":"2024-04-26T04:41:29.456901+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:33:49.567831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294371+0000","last_change":"2024-04-26T04:40:44.864369+0000","last_active":"2024-04-26T04:41:29.294371+0000","last_peered":"2024-04-26T04:41:29.294371+0000","last_clean":"2024-04-26T04:41:29.294371+0000","last_became_active":"2024-04-26T04:40:44.864062+0000","last_became_peered":"2024-04-26T04:40:44.864062+0000","last_unstale":"2024-04-26T04:41:29.294371+0000","last_undegraded":"2024-04-26T04:41:29.294371+0000","last_fullsized":"2024-04-26T04:41:29.294371+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:21:41.835888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457645+0000","last_change":"2024-04-26T04:40:44.270046+0000","last_active":"2024-04-26T04:41:29.457645+0000","last_peered":"2024-04-26T04:41:29.457645+0000","last_clean":"2024-04-26T04:41:29.457645+0000","last_became_active":"2024-04-26T04:40:44.269776+0000","last_became_peered":"2024-04-26T04:40:44.269776+0000","last_unstale":"2024-04-26T04:41:29.457645+0000","last_undegraded":"2024-04-26T04:41:29.457645+0000","last_fullsized":"2024-04-26T04:41:29.457645+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:38:17.750585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.297895+0000","last_change":"2024-04-26T04:40:44.294891+0000","last_active":"2024-04-26T04:41:29.297895+0000","last_peered":"2024-04-26T04:41:29.297895+0000","last_clean":"2024-04-26T04:41:29.297895+0000","last_became_active":"2024-04-26T04:40:44.294556+0000","last_became_peered":"2024-04-26T04:40:44.294556+0000","last_unstale":"2024-04-26T04:41:29.297895+0000","last_undegraded":"2024-04-26T04:41:29.297895+0000","last_fullsized":"2024-04-26T04:41:29.297895+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:57:20.227140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"67'6","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.298198+0000","last_change":"2024-04-26T04:40:44.272084+0000","last_active":"2024-04-26T04:41:29.298198+0000","last_peered":"2024-04-26T04:41:29.298198+0000","last_clean":"2024-04-26T04:41:29.298198+0000","last_became_active":"2024-04-26T04:40:44.271806+0000","last_became_peered":"2024-04-26T04:40:44.271806+0000","last_unstale":"2024-04-26T04:41:29.298198+0000","last_undegraded":"2024-04-26T04:41:29.298198+0000","last_fullsized":"2024-04-26T04:41:29.298198+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:42:50.578965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294544+0000","last_change":"2024-04-26T04:40:44.299343+0000","last_active":"2024-04-26T04:41:29.294544+0000","last_peered":"2024-04-26T04:41:29.294544+0000","last_clean":"2024-04-26T04:41:29.294544+0000","last_became_active":"2024-04-26T04:40:44.299081+0000","last_became_peered":"2024-04-26T04:40:44.299081+0000","last_unstale":"2024-04-26T04:41:29.294544+0000","last_undegraded":"2024-04-26T04:41:29.294544+0000","last_fullsized":"2024-04-26T04:41:29.294544+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:46:17.439373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457656+0000","last_change":"2024-04-26T04:40:44.267055+0000","last_active":"2024-04-26T04:41:29.457656+0000","last_peered":"2024-04-26T04:41:29.457656+0000","last_clean":"2024-04-26T04:41:29.457656+0000","last_became_active":"2024-04-26T04:40:44.266753+0000","last_became_peered":"2024-04-26T04:40:44.266753+0000","last_unstale":"2024-04-26T04:41:29.457656+0000","last_undegraded":"2024-04-26T04:41:29.457656+0000","last_fullsized":"2024-04-26T04:41:29.457656+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:16:33.130853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.293956+0000","last_change":"2024-04-26T04:40:44.865760+0000","last_active":"2024-04-26T04:41:29.293956+0000","last_peered":"2024-04-26T04:41:29.293956+0000","last_clean":"2024-04-26T04:41:29.293956+0000","last_became_active":"2024-04-26T04:40:44.865464+0000","last_became_peered":"2024-04-26T04:40:44.865464+0000","last_unstale":"2024-04-26T04:41:29.293956+0000","last_undegraded":"2024-04-26T04:41:29.293956+0000","last_fullsized":"2024-04-26T04:41:29.293956+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:52:36.042764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294340+0000","last_change":"2024-04-26T04:40:44.274518+0000","last_active":"2024-04-26T04:41:29.294340+0000","last_peered":"2024-04-26T04:41:29.294340+0000","last_clean":"2024-04-26T04:41:29.294340+0000","last_became_active":"2024-04-26T04:40:44.274180+0000","last_became_peered":"2024-04-26T04:40:44.274180+0000","last_unstale":"2024-04-26T04:41:29.294340+0000","last_undegraded":"2024-04-26T04:41:29.294340+0000","last_fullsized":"2024-04-26T04:41:29.294340+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:16:23.377738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294271+0000","last_change":"2024-04-26T04:40:44.293162+0000","last_active":"2024-04-26T04:41:29.294271+0000","last_peered":"2024-04-26T04:41:29.294271+0000","last_clean":"2024-04-26T04:41:29.294271+0000","last_became_active":"2024-04-26T04:40:44.292910+0000","last_became_peered":"2024-04-26T04:40:44.292910+0000","last_unstale":"2024-04-26T04:41:29.294271+0000","last_undegraded":"2024-04-26T04:41:29.294271+0000","last_fullsized":"2024-04-26T04:41:29.294271+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:17:45.414448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457088+0000","last_change":"2024-04-26T04:40:44.271984+0000","last_active":"2024-04-26T04:41:29.457088+0000","last_peered":"2024-04-26T04:41:29.457088+0000","last_clean":"2024-04-26T04:41:29.457088+0000","last_became_active":"2024-04-26T04:40:44.271731+0000","last_became_peered":"2024-04-26T04:40:44.271731+0000","last_unstale":"2024-04-26T04:41:29.457088+0000","last_undegraded":"2024-04-26T04:41:29.457088+0000","last_fullsized":"2024-04-26T04:41:29.457088+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:44:05.726631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.300456+0000","last_change":"2024-04-26T04:40:44.294330+0000","last_active":"2024-04-26T04:41:29.300456+0000","last_peered":"2024-04-26T04:41:29.300456+0000","last_clean":"2024-04-26T04:41:29.300456+0000","last_became_active":"2024-04-26T04:40:44.294017+0000","last_became_peered":"2024-04-26T04:40:44.294017+0000","last_unstale":"2024-04-26T04:41:29.300456+0000","last_undegraded":"2024-04-26T04:41:29.300456+0000","last_fullsized":"2024-04-26T04:41:29.300456+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:24:45.587863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294349+0000","last_change":"2024-04-26T04:40:44.267340+0000","last_active":"2024-04-26T04:41:29.294349+0000","last_peered":"2024-04-26T04:41:29.294349+0000","last_clean":"2024-04-26T04:41:29.294349+0000","last_became_active":"2024-04-26T04:40:44.267083+0000","last_became_peered":"2024-04-26T04:40:44.267083+0000","last_unstale":"2024-04-26T04:41:29.294349+0000","last_undegraded":"2024-04-26T04:41:29.294349+0000","last_fullsized":"2024-04-26T04:41:29.294349+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:05:59.489560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.457057+0000","last_change":"2024-04-26T04:40:44.865222+0000","last_active":"2024-04-26T04:41:29.457057+0000","last_peered":"2024-04-26T04:41:29.457057+0000","last_clean":"2024-04-26T04:41:29.457057+0000","last_became_active":"2024-04-26T04:40:44.864827+0000","last_became_peered":"2024-04-26T04:40:44.864827+0000","last_unstale":"2024-04-26T04:41:29.457057+0000","last_undegraded":"2024-04-26T04:41:29.457057+0000","last_fullsized":"2024-04-26T04:41:29.457057+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:18:54.156267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.293775+0000","last_change":"2024-04-26T04:40:44.274169+0000","last_active":"2024-04-26T04:41:29.293775+0000","last_peered":"2024-04-26T04:41:29.293775+0000","last_clean":"2024-04-26T04:41:29.293775+0000","last_became_active":"2024-04-26T04:40:44.273904+0000","last_became_peered":"2024-04-26T04:40:44.273904+0000","last_unstale":"2024-04-26T04:41:29.293775+0000","last_undegraded":"2024-04-26T04:41:29.293775+0000","last_fullsized":"2024-04-26T04:41:29.293775+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:43:11.200334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.294246+0000","last_change":"2024-04-26T04:40:44.293412+0000","last_active":"2024-04-26T04:41:29.294246+0000","last_peered":"2024-04-26T04:41:29.294246+0000","last_clean":"2024-04-26T04:41:29.294246+0000","last_became_active":"2024-04-26T04:40:44.293111+0000","last_became_peered":"2024-04-26T04:40:44.293111+0000","last_unstale":"2024-04-26T04:41:29.294246+0000","last_undegraded":"2024-04-26T04:41:29.294246+0000","last_fullsized":"2024-04-26T04:41:29.294246+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:11:55.279941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-26T04:41:29.298493+0000","last_change":"2024-04-26T04:40:44.268160+0000","last_active":"2024-04-26T04:41:29.298493+0000","last_peered":"2024-04-26T04:41:29.298493+0000","last_clean":"2024-04-26T04:41:29.298493+0000","last_became_active":"2024-04-26T04:40:44.267832+0000","last_became_peered":"2024-04-26T04:40:44.267832+0000","last_unstale":"2024-04-26T04:41:29.298493+0000","last_undegraded":"2024-04-26T04:41:29.298493+0000","last_fullsized":"2024-04-26T04:41:29.298493+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:44:23.755059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":5,"up_from":38,"seq":163208757295,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":696,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":712704,"data_stored":307054,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:41:09 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.51200000000000001,"15min":0.50800000000000001},"min":{"1min":0.311,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.75,"5min":0.877,"15min":0.877},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.502,"15min":0.49099999999999999},"min":{"1min":0.32800000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.66600000000000004,"5min":0.747,"15min":0.747},"last":0.496}]},{"osd":1,"last update":"Fri Apr 26 04:41:09 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.52900000000000003,"15min":0.53700000000000003},"min":{"1min":0.375,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.74299999999999999,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51300000000000001,"15min":0.50800000000000001},"min":{"1min":0.36199999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.70399999999999996,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.63200000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:41:09 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.51200000000000001,"15min":0.51300000000000001},"min":{"1min":0.35199999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.63800000000000001,"5min":0.88,"15min":0.88},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.54400000000000004,"15min":0.54300000000000004},"min":{"1min":0.372,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.71499999999999997,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.59399999999999997}]},{"osd":3,"last update":"Fri Apr 26 04:41:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.53800000000000003,"15min":0.54400000000000004},"min":{"1min":0.42099999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.67500000000000004,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.53500000000000003,"15min":0.52300000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.72999999999999998,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.52600000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:41:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.438,"15min":0.40799999999999997},"min":{"1min":0.33600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.70099999999999996,"5min":1.103,"15min":1.103},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.443,"15min":0.41499999999999998},"min":{"1min":0.38700000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.63100000000000001,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.48299999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:41:37 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.46899999999999997,"15min":0.45200000000000001},"min":{"1min":0.32000000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.71999999999999997,"5min":1.159,"15min":1.159},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.52000000000000002,"15min":0.5},"min":{"1min":0.40699999999999997,"5min":0.307,"15min":0.307},"max":{"1min":0.65800000000000003,"5min":1.133,"15min":1.133},"last":0.40200000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:41:00 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.52300000000000002,"15min":0.51900000000000002},"min":{"1min":0.36699999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.71599999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.56699999999999995,"15min":0.55900000000000005},"min":{"1min":0.45100000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.76300000000000001,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.374}]}]},{"osd":1,"up_from":13,"seq":55834574915,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":688,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":704512,"data_stored":306414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:41:37 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.36599999999999999,"15min":0.34100000000000003},"min":{"1min":0.24199999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.375,"15min":0.35699999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.47999999999999998,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.39200000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.39000000000000001,"15min":0.38300000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.372,"5min":0.38400000000000001,"15min":0.378},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.52600000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:41:32 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.40899999999999997,"15min":0.41999999999999998},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.66300000000000003,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.372},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.42299999999999999,"15min":0.42899999999999999},"min":{"1min":0.307,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.58699999999999997,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.35399999999999998}]},{"osd":4,"last update":"Fri Apr 26 04:40:52 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56599999999999995,"15min":0.56899999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.751,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.57399999999999995}]},{"osd":5,"last update":"Fri Apr 26 04:41:12 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.61599999999999999,"15min":0.628},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.81000000000000005,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.61299999999999999,"15min":0.627},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.83699999999999997,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.46500000000000002}]},{"osd":6,"last update":"Fri Apr 26 04:41:37 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.60899999999999999,"15min":0.62},"min":{"1min":0.45800000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.71599999999999997,"5min":0.879,"15min":0.879},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.624,"5min":0.63200000000000001,"15min":0.65000000000000002},"min":{"1min":0.44400000000000001,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.86299999999999999,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.499}]},{"osd":7,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.57299999999999995,"15min":0.57499999999999996},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.73099999999999998,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.59399999999999997,"15min":0.59199999999999997},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.54900000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411389,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298052,"kb_used_data":696,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447100,"statfs":{"total":95995035648,"available":95689830400,"internally_reserved":0,"allocated":712704,"data_stored":307018,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:41:04 2024","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.375,"15min":0.36499999999999999},"min":{"1min":0.23100000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.49199999999999999,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.378},{"interface":"front","average":{"1min":0.33600000000000002,"5min":0.39100000000000001,"15min":0.39200000000000002},"min":{"1min":0.23599999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.55500000000000005,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.29699999999999999}]},{"osd":1,"last update":"Fri Apr 26 04:41:04 2024","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.38800000000000001,"15min":0.38200000000000001},"min":{"1min":0.24399999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.53000000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.41299999999999998,"15min":0.40899999999999997},"min":{"1min":0.27900000000000003,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.54200000000000004,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.36199999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:41:27 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.45000000000000001,"15min":0.46700000000000003},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.58699999999999997,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.46899999999999997,"15min":0.497},"min":{"1min":0.28799999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.51700000000000002,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.33500000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:40:48 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.56399999999999995,"15min":0.56899999999999995},"min":{"1min":0.42899999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59699999999999998,"15min":0.60099999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.83999999999999997,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.63800000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:41:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.57599999999999996,"15min":0.57299999999999995},"min":{"1min":0.442,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.55500000000000005,"15min":0.55000000000000004},"min":{"1min":0.40799999999999997,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.71499999999999997}]},{"osd":6,"last update":"Fri Apr 26 04:41:39 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.58099999999999996,"15min":0.56000000000000005},"min":{"1min":0.41799999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.84799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.58999999999999997,"15min":0.57799999999999996},"min":{"1min":0.46500000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.81000000000000005,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.69599999999999995}]},{"osd":7,"last update":"Fri Apr 26 04:41:04 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.61899999999999999,"15min":0.621},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.872,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.628},{"interface":"front","average":{"1min":0.621,"5min":0.62,"15min":0.62},"min":{"1min":0.48699999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.81399999999999995,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.66100000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298632,"kb_used_data":1276,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446520,"statfs":{"total":95995035648,"available":95689236480,"internally_reserved":0,"allocated":1306624,"data_stored":897374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:41:41 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.36699999999999999,"15min":0.34200000000000003},"min":{"1min":0.219,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.40400000000000003,"15min":0.36399999999999999},"min":{"1min":0.248,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.482,"5min":1.482,"15min":1.482},"last":0.29699999999999999}]},{"osd":2,"last update":"Fri Apr 26 04:41:03 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.42699999999999999,"15min":0.40500000000000003},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.655,"5min":1.655,"15min":1.655},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.40400000000000003,"15min":0.39500000000000002},"min":{"1min":0.26500000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.28399999999999997}]},{"osd":3,"last update":"Fri Apr 26 04:41:30 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47999999999999998,"15min":0.48999999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.524,"5min":1.524,"15min":1.524},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.44,"5min":0.44400000000000001,"15min":0.45600000000000002},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.3560000000000001,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":0.34499999999999997}]},{"osd":4,"last update":"Fri Apr 26 04:40:48 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.57799999999999996,"15min":0.58499999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.497},{"interface":"front","average":{"1min":0.622,"5min":0.60399999999999998,"15min":0.60599999999999998},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.5700000000000001,"5min":1.5700000000000001,"15min":1.5700000000000001},"last":0.443}]},{"osd":5,"last update":"Fri Apr 26 04:41:14 2024","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.59699999999999998,"15min":0.59999999999999998},"min":{"1min":0.45800000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.4239999999999999,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.64600000000000002,"15min":0.65500000000000003},"min":{"1min":0.44900000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.599,"5min":1.599,"15min":1.599},"last":0.47999999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.58699999999999997,"15min":0.58299999999999996},"min":{"1min":0.45600000000000002,"5min":0.379,"15min":0.379},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.64800000000000002,"15min":0.65500000000000003},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.6279999999999999,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":0.42499999999999999}]},{"osd":7,"last update":"Fri Apr 26 04:41:03 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.56499999999999995,"15min":0.55500000000000005},"min":{"1min":0.42899999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.3879999999999999,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.59299999999999997,"15min":0.58099999999999996},"min":{"1min":0.46800000000000003,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.52800000000000002}]}]},{"osd":7,"up_from":54,"seq":231928234021,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298648,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446504,"statfs":{"total":95995035648,"available":95689220096,"internally_reserved":0,"allocated":1323008,"data_stored":898227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.51700000000000002,"15min":0.51600000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.53100000000000003,"15min":0.53400000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.71699999999999997,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.505}]},{"osd":1,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.54100000000000004,"15min":0.53900000000000003},"min":{"1min":0.32100000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.504,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.45800000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.56299999999999994,"15min":0.56899999999999995},"min":{"1min":0.40500000000000003,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.54200000000000004,"15min":0.54600000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.58399999999999996}]},{"osd":3,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55900000000000005,"15min":0.56000000000000005},"min":{"1min":0.42499999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.78800000000000003,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.58099999999999996,"15min":0.58599999999999997},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.76800000000000002,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.67500000000000004}]},{"osd":4,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.52000000000000002,"15min":0.52100000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.1160000000000001,"5min":1.1160000000000001,"15min":1.1160000000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54100000000000004,"15min":0.53900000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.70699999999999996,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.39800000000000002}]},{"osd":5,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.52800000000000002,"15min":0.52600000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.71899999999999997,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.55900000000000005,"15min":0.56100000000000005},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.755,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.66100000000000003}]},{"osd":6,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.57499999999999996,"15min":0.57099999999999995},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":1.046,"5min":1.046,"15min":1.046},"last":0.622},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.61199999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182457,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298052,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447100,"statfs":{"total":95995035648,"available":95689830400,"internally_reserved":0,"allocated":708608,"data_stored":306504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:41:26 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.373,"15min":0.38100000000000001},"min":{"1min":0.245,"5min":0.219,"15min":0.219},"max":{"1min":0.67900000000000005,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.39200000000000002,"15min":0.40300000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.61499999999999999,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.32600000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:41:26 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.42199999999999999,"15min":0.443},"min":{"1min":0.26100000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.58099999999999996,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.255},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.42699999999999999,"15min":0.437},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":0.34000000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:41:26 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.437,"15min":0.45300000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.2070000000000001,"5min":1.2070000000000001,"15min":1.2070000000000001},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.41399999999999998,"15min":0.41999999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.242,"5min":1.242,"15min":1.242},"last":0.35899999999999999}]},{"osd":4,"last update":"Fri Apr 26 04:40:46 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.42999999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.82999999999999996,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.56000000000000005,"15min":0.55900000000000005},"min":{"1min":0.41699999999999998,"5min":0.373,"15min":0.373},"max":{"1min":0.79000000000000004,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.78500000000000003}]},{"osd":5,"last update":"Fri Apr 26 04:41:08 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56200000000000006,"15min":0.56799999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56699999999999995,"15min":0.56599999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.76800000000000002}]},{"osd":6,"last update":"Fri Apr 26 04:41:41 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.57099999999999995,"15min":0.55500000000000005},"min":{"1min":0.41799999999999998,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.745},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.57299999999999995,"15min":0.56299999999999994},"min":{"1min":0.40999999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.81100000000000005,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.64100000000000001}]},{"osd":7,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.55300000000000005,"15min":0.55100000000000005},"min":{"1min":0.42699999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.71399999999999997,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.57199999999999995,"15min":0.56599999999999995},"min":{"1min":0.441,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.075,"5min":1.075,"15min":1.075},"last":0.70699999999999996}]}]},{"osd":4,"up_from":30,"seq":128849018932,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":729088,"data_stored":308112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:48 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.52500000000000002,"15min":0.51100000000000001},"min":{"1min":0.41099999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.80100000000000005,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.52300000000000002,"15min":0.51500000000000001},"min":{"1min":0.40400000000000003,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.41899999999999998}]},{"osd":1,"last update":"Fri Apr 26 04:40:48 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.33500000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.53200000000000003,"15min":0.51800000000000002},"min":{"1min":0.41899999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.70499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.379}]},{"osd":2,"last update":"Fri Apr 26 04:40:48 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.54900000000000004,"15min":0.54100000000000004},"min":{"1min":0.39800000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.753,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.54300000000000004,"15min":0.54100000000000004},"min":{"1min":0.435,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.78900000000000003,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.46100000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:40:48 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.45100000000000001,"5min":0.372,"15min":0.372},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.501},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.53100000000000003,"15min":0.52500000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.61399999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:41:10 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.46300000000000002,"15min":0.45500000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.72599999999999998,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.497,"5min":0.46800000000000003,"15min":0.45000000000000001},"min":{"1min":0.35099999999999998,"5min":0.318,"15min":0.318},"max":{"1min":0.65900000000000003,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.54200000000000004}]},{"osd":6,"last update":"Fri Apr 26 04:41:35 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.5,"15min":0.48799999999999999},"min":{"1min":0.314,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.68500000000000005,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.53400000000000003,"15min":0.53000000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.75700000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.60099999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56699999999999995,"15min":0.56799999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.627},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.53200000000000003,"15min":0.53600000000000003},"min":{"1min":0.38200000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.71099999999999997}]}]},{"osd":6,"up_from":46,"seq":197568495658,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298640,"kb_used_data":1280,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446512,"statfs":{"total":95995035648,"available":95689228288,"internally_reserved":0,"allocated":1310720,"data_stored":898131,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.33900000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.67700000000000005,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.51400000000000001,"15min":0.52600000000000002},"min":{"1min":0.36799999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.58199999999999996,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.52800000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52600000000000002,"15min":0.52900000000000003},"min":{"1min":0.39600000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.64300000000000002,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.627},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.55700000000000005,"15min":0.57499999999999996},"min":{"1min":0.36099999999999999,"5min":0.317,"15min":0.317},"max":{"1min":0.62,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.45500000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.53700000000000003,"15min":0.54200000000000004},"min":{"1min":0.39800000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.64300000000000002,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.54900000000000004,"15min":0.55200000000000005},"min":{"1min":0.45600000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.67900000000000005,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.55400000000000005}]},{"osd":3,"last update":"Fri Apr 26 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.56999999999999995,"15min":0.57899999999999996},"min":{"1min":0.47699999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.69299999999999995,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.505},{"interface":"front","average":{"1min":0.505,"5min":0.52800000000000002,"15min":0.53400000000000003},"min":{"1min":0.39700000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.627,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.57699999999999996}]},{"osd":4,"last update":"Fri Apr 26 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.437,"15min":0.42999999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.504,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.47999999999999998,"15min":0.47499999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.66500000000000004,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.56799999999999995}]},{"osd":5,"last update":"Fri Apr 26 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.45800000000000002,"15min":0.45200000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.65300000000000002,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51400000000000001,"15min":0.503},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.71199999999999997,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.60299999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:40:59 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.34000000000000002,"5min":0.312,"15min":0.312},"max":{"1min":0.73699999999999999,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.56499999999999995,"15min":0.56399999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.75800000000000001,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.65800000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:41:47.850 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-26T04:41:47.850 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-26T04:41:47.850 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-26T04:41:47.851 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:41:48.102 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:48.571 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:48 smithi156 ceph-mon[27570]: from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:41:48.572 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:48 smithi156 ceph-mon[27570]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:48.572 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:48 smithi156 ceph-mon[39130]: from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:41:48.572 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:48 smithi156 ceph-mon[39130]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:48.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:48 smithi171 ceph-mon[31139]: from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:41:48.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:48 smithi171 ceph-mon[31139]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:50.135 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:41:50.135 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-26T04:41:50.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:50 smithi156 ceph-mon[27570]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:50.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:50 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2830918481' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:41:50.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:50 smithi156 ceph-mon[39130]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:50.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:50 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2830918481' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:41:50.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:50 smithi171 ceph-mon[31139]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:50.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:50 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2830918481' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:41:50.757 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-26T04:41:50.758 INFO:teuthology.run_tasks:Running task print... 2024-04-26T04:41:50.790 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-04-26T04:41:50.790 INFO:teuthology.run_tasks:Running task parallel... 2024-04-26T04:41:50.799 INFO:teuthology.task.parallel:starting parallel... 2024-04-26T04:41:50.800 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-26T04:41:50.801 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-04-26T04:41:50.808 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-04-26T04:41:50.809 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-04-26T04:41:50.809 DEBUG:teuthology.orchestra.run.smithi156:> stat -- /home/ubuntu/cephtest/mnt.0 2024-04-26T04:41:50.809 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-26T04:41:50.809 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-26T04:41:50.810 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi156.front.sepia.ceph.com 2024-04-26T04:41:50.810 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-04-26T04:41:50.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:41:50.825 INFO:teuthology.orchestra.run.smithi156.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-04-26T04:41:50.826 DEBUG:teuthology.orchestra.run.smithi156:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-04-26T04:41:50.879 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-04-26T04:41:50.880 DEBUG:teuthology.orchestra.run.smithi156:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-04-26T04:41:50.935 INFO:tasks.workunit:timeout=3h 2024-04-26T04:41:50.935 INFO:tasks.workunit:cleanup=True 2024-04-26T04:41:50.935 DEBUG:teuthology.orchestra.run.smithi156:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-04-26T04:41:50.990 INFO:tasks.workunit.client.0.smithi156.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-04-26T04:41:51.067 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:52.573 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:52 smithi156 ceph-mon[27570]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:52.574 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:52 smithi156 ceph-mon[39130]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:52.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:52 smithi171 ceph-mon[31139]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:53.298 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:41:53 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:41:53] ENGINE Bus STOPPING 2024-04-26T04:41:53.298 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:41:53 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:41:53] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:41:53.298 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:41:53 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:41:53] ENGINE Bus STOPPED 2024-04-26T04:41:53.298 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:41:53 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:41:53] ENGINE Bus STARTING 2024-04-26T04:41:53.299 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:41:53 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:41:53] ENGINE Serving on http://:::9283 2024-04-26T04:41:53.299 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:41:53 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:41:53] ENGINE Bus STARTED 2024-04-26T04:41:53.732 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-26T04:41:54.043 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:54.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:54 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/830971548' entity='client.admin' 2024-04-26T04:41:54.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:54 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:54.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:54 smithi171 ceph-mon[31139]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:54.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:54 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:54.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:54 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:54.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:54 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:54.281 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:54 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/830971548' entity='client.admin' 2024-04-26T04:41:54.282 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:54 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:54.282 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:54 smithi156 ceph-mon[27570]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:54.282 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:54 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:54.282 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:54 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:54.282 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:54 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:54.282 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:54 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/830971548' entity='client.admin' 2024-04-26T04:41:54.282 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:54 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:54.282 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:54 smithi156 ceph-mon[39130]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:54.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:54 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:54.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:54 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:54.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:54 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:41:54.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:41:54 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:41:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:41:56.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:56 smithi156 ceph-mon[27570]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:56.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:56 smithi156 ceph-mon[39130]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:56.594 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:41:56 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:41:56] "GET /metrics HTTP/1.1" 200 239117 "" "Prometheus/2.43.0" 2024-04-26T04:41:56.717 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-26T04:41:56.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:56 smithi171 ceph-mon[31139]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:56.973 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:58.413 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:58 smithi156 ceph-mon[27570]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:58.413 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:58 smithi156 ceph-mon[39130]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:58.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:58 smithi171 ceph-mon[31139]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:58.897 INFO:teuthology.orchestra.run.smithi156.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:41:59.588 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:41:59.857 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:42:00.167 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[27570]: from='client.24707 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:42:00.167 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[27570]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:42:00.167 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:00.167 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:00.168 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:00.168 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:42:00.168 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:00.168 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[27570]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:42:00.168 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[27570]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:00.168 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:00.168 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[39130]: from='client.24707 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:42:00.169 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[39130]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:42:00.169 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:00.169 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:00.169 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:00.169 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:42:00.169 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:00.169 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[39130]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:42:00.169 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[39130]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:00.169 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:41:59 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:00.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:59 smithi171 ceph-mon[31139]: from='client.24707 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:42:00.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:59 smithi171 ceph-mon[31139]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:42:00.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:59 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:00.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:59 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:00.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:59 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:00.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:59 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:42:00.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:59 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:00.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:59 smithi171 ceph-mon[31139]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:42:00.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:59 smithi171 ceph-mon[31139]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:00.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:41:59 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:02.023 INFO:teuthology.orchestra.run.smithi156.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:42:02.023 INFO:teuthology.orchestra.run.smithi156.stdout:alertmanager.a smithi156 *:9093,9094 running (85s) 57s ago 2m 17.3M - 0.25.0 c8568f914cd2 9742e32fd5fc 2024-04-26T04:42:02.023 INFO:teuthology.orchestra.run.smithi156.stdout:grafana.a smithi171 *:3000 running (2m) 60s ago 2m 81.9M - 9.4.7 954c08fa6188 d42cfd971c1c 2024-04-26T04:42:02.023 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi156.wyfzii smithi156 running (72s) 57s ago 71s 13.0M - 17.2.7-902-gc61472c8 0de407c1b3af f42880bb83a3 2024-04-26T04:42:02.023 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi171.kzjlqn smithi171 running (69s) 60s ago 69s 12.0M - 17.2.7-902-gc61472c8 0de407c1b3af 31b518bc187c 2024-04-26T04:42:02.024 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.x smithi171 *:8443 running (6m) 60s ago 6m 433M - 17.2.7-902-gc61472c8 0de407c1b3af 77edd00db79e 2024-04-26T04:42:02.024 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.y smithi156 *:9283 running (8m) 57s ago 8m 477M - 17.2.7-902-gc61472c8 0de407c1b3af b232ee6793f9 2024-04-26T04:42:02.024 INFO:teuthology.orchestra.run.smithi156.stdout:mon.a smithi156 running (8m) 57s ago 8m 41.6M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 3d899f93d0ef 2024-04-26T04:42:02.024 INFO:teuthology.orchestra.run.smithi156.stdout:mon.b smithi171 running (6m) 60s ago 6m 35.8M 2048M 17.2.7-902-gc61472c8 0de407c1b3af fb359e8b680c 2024-04-26T04:42:02.024 INFO:teuthology.orchestra.run.smithi156.stdout:mon.c smithi156 running (6m) 57s ago 6m 36.6M 2048M 17.2.7-902-gc61472c8 0de407c1b3af fc34a850791e 2024-04-26T04:42:02.024 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.a smithi156 *:9100 running (110s) 57s ago 105s 8338k - 1.5.0 0da6a335fe13 71daaa840a85 2024-04-26T04:42:02.024 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.b smithi171 *:9100 running (102s) 60s ago 101s 8116k - 1.5.0 0da6a335fe13 5ef699e7d857 2024-04-26T04:42:02.024 INFO:teuthology.orchestra.run.smithi156.stdout:osd.0 smithi156 running (6m) 57s ago 6m 42.2M 4016M 17.2.7-902-gc61472c8 0de407c1b3af e1273966ada0 2024-04-26T04:42:02.024 INFO:teuthology.orchestra.run.smithi156.stdout:osd.1 smithi156 running (5m) 57s ago 5m 41.6M 4016M 17.2.7-902-gc61472c8 0de407c1b3af 7abca656e994 2024-04-26T04:42:02.024 INFO:teuthology.orchestra.run.smithi156.stdout:osd.2 smithi156 running (5m) 57s ago 5m 40.5M 4016M 17.2.7-902-gc61472c8 0de407c1b3af c951e2171c5d 2024-04-26T04:42:02.024 INFO:teuthology.orchestra.run.smithi156.stdout:osd.3 smithi156 running (4m) 57s ago 4m 40.4M 4016M 17.2.7-902-gc61472c8 0de407c1b3af a2c01ca7770f 2024-04-26T04:42:02.024 INFO:teuthology.orchestra.run.smithi156.stdout:osd.4 smithi171 running (4m) 60s ago 4m 43.8M 4016M 17.2.7-902-gc61472c8 0de407c1b3af 388a24a70001 2024-04-26T04:42:02.024 INFO:teuthology.orchestra.run.smithi156.stdout:osd.5 smithi171 running (4m) 60s ago 4m 40.4M 4016M 17.2.7-902-gc61472c8 0de407c1b3af 703235a355b5 2024-04-26T04:42:02.024 INFO:teuthology.orchestra.run.smithi156.stdout:osd.6 smithi171 running (3m) 60s ago 3m 43.3M 4016M 17.2.7-902-gc61472c8 0de407c1b3af 3dad877f8247 2024-04-26T04:42:02.025 INFO:teuthology.orchestra.run.smithi156.stdout:osd.7 smithi171 running (3m) 60s ago 3m 40.7M 4016M 17.2.7-902-gc61472c8 0de407c1b3af e275b2da9b18 2024-04-26T04:42:02.025 INFO:teuthology.orchestra.run.smithi156.stdout:prometheus.a smithi171 *:9095 running (80s) 60s ago 2m 35.2M - 2.43.0 a07b618ecd1d be422f8a7019 2024-04-26T04:42:02.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:02 smithi156 ceph-mon[27570]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:02.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:02 smithi156 ceph-mon[39130]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:02.664 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for minority of mons to upgrade' 2024-04-26T04:42:02.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:02 smithi171 ceph-mon[31139]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:02.960 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:42:03.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:03 smithi156 ceph-mon[27570]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:42:03.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:03 smithi156 ceph-mon[39130]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:42:03.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:03 smithi171 ceph-mon[31139]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:42:04.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:04 smithi156 ceph-mon[27570]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:04.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:04 smithi156 ceph-mon[39130]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:04.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:04 smithi171 ceph-mon[31139]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:04.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:42:04 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:42:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:42:05.243 INFO:teuthology.orchestra.run.smithi156.stdout:wait for minority of mons to upgrade 2024-04-26T04:42:05.710 DEBUG:teuthology.orchestra.run.smithi156:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-04-26T04:42:05.736 INFO:tasks.workunit.client.0.smithi156.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-04-26T04:42:05.738 INFO:tasks.workunit.client.0.smithi156.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-04-26T04:42:05.738 INFO:tasks.workunit.client.0.smithi156.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-04-26T04:42:06.180 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-04-26T04:42:06.494 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:42:06.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:06 smithi156 ceph-mon[27570]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:06.593 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:42:06 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:42:06] "GET /metrics HTTP/1.1" 200 239101 "" "Prometheus/2.43.0" 2024-04-26T04:42:06.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:06 smithi156 ceph-mon[39130]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:06.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:06 smithi171 ceph-mon[31139]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:07.014 INFO:tasks.workunit.client.0.smithi156.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-04-26T04:42:07.130 INFO:tasks.workunit.client.0.smithi156.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-04-26T04:42:07.170 INFO:tasks.workunit.client.0.smithi156.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-04-26T04:42:07.175 INFO:tasks.workunit.client.0.smithi156.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-04-26T04:42:07.175 INFO:tasks.workunit.client.0.smithi156.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-04-26T04:42:07.211 INFO:tasks.workunit.client.0.smithi156.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-04-26T04:42:07.214 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-04-26T04:42:07.214 DEBUG:teuthology.orchestra.run.smithi156:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-04-26T04:42:07.230 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2024-04-26T04:42:07.231 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-04-26T04:42:07.231 DEBUG:teuthology.orchestra.run.smithi156:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-04-26T04:42:07.296 INFO:tasks.workunit.client.0.smithi156.stdout:Running main() from gmock_main.cc 2024-04-26T04:42:07.296 INFO:tasks.workunit.client.0.smithi156.stdout:[==========] Running 70 tests from 1 test suite. 2024-04-26T04:42:07.296 INFO:tasks.workunit.client.0.smithi156.stdout:[----------] Global test environment set-up. 2024-04-26T04:42:07.296 INFO:tasks.workunit.client.0.smithi156.stdout:[----------] 70 tests from TestClsRbd 2024-04-26T04:42:08.456 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:08 smithi156 ceph-mon[27570]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:08.456 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:08 smithi156 ceph-mon[39130]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:08.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:08 smithi171 ceph-mon[31139]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:09.307 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.get_all_features 2024-04-26T04:42:09.325 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.get_all_features (19 ms) 2024-04-26T04:42:09.325 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.copyup 2024-04-26T04:42:09.509 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.copyup (183 ms) 2024-04-26T04:42:09.509 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-04-26T04:42:09.521 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.sparse_copyup (13 ms) 2024-04-26T04:42:09.521 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-04-26T04:42:09.529 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2024-04-26T04:42:09.529 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-04-26T04:42:09.540 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.add_remove_child (12 ms) 2024-04-26T04:42:09.540 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.directory_methods 2024-04-26T04:42:09.563 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.directory_methods (23 ms) 2024-04-26T04:42:09.563 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.create 2024-04-26T04:42:09.575 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.create (12 ms) 2024-04-26T04:42:09.575 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.get_features 2024-04-26T04:42:09.577 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2024-04-26T04:42:09.577 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-04-26T04:42:09.578 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.get_object_prefix (1 ms) 2024-04-26T04:42:09.578 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-04-26T04:42:09.580 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2024-04-26T04:42:09.580 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-04-26T04:42:09.581 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2024-04-26T04:42:09.581 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-04-26T04:42:09.582 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-04-26T04:42:09.583 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-04-26T04:42:09.586 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2024-04-26T04:42:09.586 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.get_size 2024-04-26T04:42:09.589 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.get_size (3 ms) 2024-04-26T04:42:09.589 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.set_size 2024-04-26T04:42:09.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:09 smithi156 ceph-mon[27570]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T04:42:09.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:09 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/701980545' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi156-84603-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:42:09.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:09 smithi156 ceph-mon[27570]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi156-84603-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:42:09.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:09 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3671455209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:09.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:09 smithi156 ceph-mon[39130]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T04:42:09.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:09 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/701980545' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi156-84603-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:42:09.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:09 smithi156 ceph-mon[39130]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi156-84603-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:42:09.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:09 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3671455209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:09.594 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.set_size (4 ms) 2024-04-26T04:42:09.594 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.protection_status 2024-04-26T04:42:09.611 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.protection_status (19 ms) 2024-04-26T04:42:09.612 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-04-26T04:42:09.619 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2024-04-26T04:42:09.619 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.parents_v1 2024-04-26T04:42:09.658 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.parents_v1 (39 ms) 2024-04-26T04:42:09.659 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.parents_v2 2024-04-26T04:42:09.685 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.parents_v2 (27 ms) 2024-04-26T04:42:09.685 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.snapshots 2024-04-26T04:42:09.704 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.snapshots (19 ms) 2024-04-26T04:42:09.704 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.snapid_race 2024-04-26T04:42:09.710 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.snapid_race (6 ms) 2024-04-26T04:42:09.710 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.stripingv2 2024-04-26T04:42:09.719 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.stripingv2 (8 ms) 2024-04-26T04:42:09.719 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.object_map_save 2024-04-26T04:42:09.720 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.object_map_save (2 ms) 2024-04-26T04:42:09.720 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-04-26T04:42:09.724 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.object_map_resize (5 ms) 2024-04-26T04:42:09.725 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.object_map_update 2024-04-26T04:42:09.728 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.object_map_update (4 ms) 2024-04-26T04:42:09.729 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-04-26T04:42:09.729 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2024-04-26T04:42:09.729 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-04-26T04:42:09.732 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.object_map_snap_add (3 ms) 2024-04-26T04:42:09.732 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-04-26T04:42:09.737 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.object_map_snap_remove (5 ms) 2024-04-26T04:42:09.737 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.flags 2024-04-26T04:42:09.741 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.flags (4 ms) 2024-04-26T04:42:09.741 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.metadata 2024-04-26T04:42:09.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:09 smithi171 ceph-mon[31139]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T04:42:09.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:09 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/701980545' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi156-84603-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:42:09.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:09 smithi171 ceph-mon[31139]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi156-84603-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:42:09.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:09 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3671455209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:09.764 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.metadata (23 ms) 2024-04-26T04:42:09.764 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.set_features 2024-04-26T04:42:09.770 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.set_features (6 ms) 2024-04-26T04:42:09.770 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.mirror 2024-04-26T04:42:09.804 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.mirror (34 ms) 2024-04-26T04:42:09.804 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.mirror_image 2024-04-26T04:42:09.818 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.mirror_image (14 ms) 2024-04-26T04:42:09.818 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-04-26T04:42:10.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:10 smithi156 ceph-mon[39130]: pgmap v93: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:10.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:10 smithi156 ceph-mon[39130]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:42:10.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:10 smithi156 ceph-mon[39130]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi156-84603-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:42:10.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:10 smithi156 ceph-mon[39130]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T04:42:10.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:10 smithi156 ceph-mon[27570]: pgmap v93: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:10.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:10 smithi156 ceph-mon[27570]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:42:10.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:10 smithi156 ceph-mon[27570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi156-84603-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:42:10.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:10 smithi156 ceph-mon[27570]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T04:42:10.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:10 smithi171 ceph-mon[31139]: pgmap v93: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:10.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:10 smithi171 ceph-mon[31139]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:42:10.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:10 smithi171 ceph-mon[31139]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi156-84603-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:42:10.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:10 smithi171 ceph-mon[31139]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T04:42:11.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:11 smithi156 ceph-mon[27570]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T04:42:11.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:11 smithi156 ceph-mon[39130]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T04:42:11.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:11 smithi171 ceph-mon[31139]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T04:42:12.014 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.mirror_image_status (2196 ms) 2024-04-26T04:42:12.014 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-04-26T04:42:12.093 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.mirror_image_map (79 ms) 2024-04-26T04:42:12.093 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-04-26T04:42:12.101 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2024-04-26T04:42:12.101 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-04-26T04:42:12.111 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.mirror_snapshot (10 ms) 2024-04-26T04:42:12.111 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-04-26T04:42:12.113 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2024-04-26T04:42:12.113 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-04-26T04:42:12.115 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2024-04-26T04:42:12.115 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-04-26T04:42:12.118 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2024-04-26T04:42:12.118 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-04-26T04:42:12.123 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_dir_rename (5 ms) 2024-04-26T04:42:12.123 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-04-26T04:42:12.126 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-04-26T04:42:12.126 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-04-26T04:42:12.129 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2024-04-26T04:42:12.130 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_image_add 2024-04-26T04:42:12.131 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_image_add (1 ms) 2024-04-26T04:42:12.131 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-04-26T04:42:12.134 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2024-04-26T04:42:12.134 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_image_list 2024-04-26T04:42:12.137 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2024-04-26T04:42:12.137 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-04-26T04:42:12.140 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2024-04-26T04:42:12.141 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.image_group_add 2024-04-26T04:42:12.142 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2024-04-26T04:42:12.143 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-04-26T04:42:12.145 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.image_group_remove (2 ms) 2024-04-26T04:42:12.145 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.image_group_get 2024-04-26T04:42:12.147 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2024-04-26T04:42:12.147 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-04-26T04:42:12.148 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2024-04-26T04:42:12.148 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-04-26T04:42:12.150 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (1 ms) 2024-04-26T04:42:12.150 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-04-26T04:42:12.153 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2024-04-26T04:42:12.153 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-04-26T04:42:12.155 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2024-04-26T04:42:12.156 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-04-26T04:42:12.157 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_snap_set (2 ms) 2024-04-26T04:42:12.157 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-04-26T04:42:12.161 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2024-04-26T04:42:12.161 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-04-26T04:42:12.174 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (13 ms) 2024-04-26T04:42:12.174 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-04-26T04:42:12.363 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (189 ms) 2024-04-26T04:42:12.363 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-04-26T04:42:12.366 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2024-04-26T04:42:12.366 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-04-26T04:42:12.368 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2024-04-26T04:42:12.368 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.trash_methods 2024-04-26T04:42:12.374 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.trash_methods (6 ms) 2024-04-26T04:42:12.374 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.op_features 2024-04-26T04:42:12.381 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.op_features (7 ms) 2024-04-26T04:42:12.381 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.clone_parent 2024-04-26T04:42:12.400 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.clone_parent (19 ms) 2024-04-26T04:42:12.400 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-04-26T04:42:12.407 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.clone_parent_ns (7 ms) 2024-04-26T04:42:12.407 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.clone_child 2024-04-26T04:42:12.424 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.clone_child (11 ms) 2024-04-26T04:42:12.424 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-04-26T04:42:12.425 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2024-04-26T04:42:12.425 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.migration 2024-04-26T04:42:12.433 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.migration (9 ms) 2024-04-26T04:42:12.434 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.migration_v1 2024-04-26T04:42:12.443 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.migration_v1 (9 ms) 2024-04-26T04:42:12.443 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-04-26T04:42:12.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:12 smithi156 ceph-mon[27570]: pgmap v96: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 823 KiB/s wr, 445 op/s 2024-04-26T04:42:12.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:12 smithi156 ceph-mon[27570]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:42:12.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:12 smithi156 ceph-mon[27570]: Cluster is now healthy 2024-04-26T04:42:12.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:12 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/759977566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:12.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:12 smithi156 ceph-mon[39130]: pgmap v96: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 823 KiB/s wr, 445 op/s 2024-04-26T04:42:12.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:12 smithi156 ceph-mon[39130]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:42:12.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:12 smithi156 ceph-mon[39130]: Cluster is now healthy 2024-04-26T04:42:12.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:12 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/759977566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:12.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:12 smithi171 ceph-mon[31139]: pgmap v96: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 823 KiB/s wr, 445 op/s 2024-04-26T04:42:12.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:12 smithi171 ceph-mon[31139]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:42:12.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:12 smithi171 ceph-mon[31139]: Cluster is now healthy 2024-04-26T04:42:12.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:12 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/759977566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:14.325 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1882 ms) 2024-04-26T04:42:14.326 INFO:tasks.workunit.client.0.smithi156.stdout:[ RUN ] TestClsRbd.sparsify 2024-04-26T04:42:14.354 INFO:tasks.workunit.client.0.smithi156.stdout:[ OK ] TestClsRbd.sparsify (29 ms) 2024-04-26T04:42:14.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:14 smithi156 ceph-mon[39130]: pgmap v97: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 823 KiB/s wr, 445 op/s 2024-04-26T04:42:14.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:14 smithi156 ceph-mon[39130]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T04:42:14.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:14 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3719632211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:14.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:14 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:14.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:14 smithi156 ceph-mon[27570]: pgmap v97: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 823 KiB/s wr, 445 op/s 2024-04-26T04:42:14.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:14 smithi156 ceph-mon[27570]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T04:42:14.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:14 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3719632211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:14.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:14 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:14.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:42:14 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:42:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:42:14.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:14 smithi171 ceph-mon[31139]: pgmap v97: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 823 KiB/s wr, 445 op/s 2024-04-26T04:42:14.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:14 smithi171 ceph-mon[31139]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T04:42:14.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:14 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3719632211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:14.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:14 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:15.332 INFO:tasks.workunit.client.0.smithi156.stdout:[----------] 70 tests from TestClsRbd (5048 ms total) 2024-04-26T04:42:15.332 INFO:tasks.workunit.client.0.smithi156.stdout: 2024-04-26T04:42:15.332 INFO:tasks.workunit.client.0.smithi156.stdout:[----------] Global test environment tear-down 2024-04-26T04:42:15.332 INFO:tasks.workunit.client.0.smithi156.stdout:[==========] 70 tests from 1 test suite ran. (8035 ms total) 2024-04-26T04:42:15.332 INFO:tasks.workunit.client.0.smithi156.stdout:[ PASSED ] 70 tests. 2024-04-26T04:42:15.334 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-04-26T04:42:15.334 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-04-26T04:42:15.403 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2024-04-26T04:42:15.403 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-04-26T04:42:15.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:15 smithi156 ceph-mon[27570]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T04:42:15.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:15 smithi156 ceph-mon[39130]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T04:42:15.635 DEBUG:teuthology.parallel:result is None 2024-04-26T04:42:15.635 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-04-26T04:42:15.660 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-04-26T04:42:15.660 DEBUG:teuthology.orchestra.run.smithi156:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-04-26T04:42:15.725 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-04-26T04:42:15.726 INFO:teuthology.task.sequential:In sequential, running task print... 2024-04-26T04:42:15.726 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2024-04-26T04:42:15.726 DEBUG:teuthology.parallel:result is None 2024-04-26T04:42:15.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:15 smithi171 ceph-mon[31139]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T04:42:16.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:16 smithi156 ceph-mon[27570]: pgmap v100: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.4 MiB/s wr, 2.48k op/s 2024-04-26T04:42:16.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:16 smithi156 ceph-mon[27570]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T04:42:16.593 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:42:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:42:16] "GET /metrics HTTP/1.1" 200 242046 "" "Prometheus/2.43.0" 2024-04-26T04:42:16.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:16 smithi156 ceph-mon[39130]: pgmap v100: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.4 MiB/s wr, 2.48k op/s 2024-04-26T04:42:16.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:16 smithi156 ceph-mon[39130]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T04:42:16.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:16 smithi171 ceph-mon[31139]: pgmap v100: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.4 MiB/s wr, 2.48k op/s 2024-04-26T04:42:16.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:16 smithi171 ceph-mon[31139]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T04:42:17.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:17 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1897677745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:17.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:17 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1897677745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:17.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:17 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1897677745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:18.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:18 smithi156 ceph-mon[27570]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:18.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:18 smithi156 ceph-mon[39130]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:18.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:18 smithi171 ceph-mon[31139]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:19.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:19 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1882886916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:19.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:19 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:19.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:19 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1882886916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:19.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:19 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:19.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:19 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1882886916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:19.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:19 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:20.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:20 smithi156 ceph-mon[27570]: pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:20.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:20 smithi156 ceph-mon[39130]: pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:20.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:20 smithi171 ceph-mon[31139]: pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:21.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:21 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/917665514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:21.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:21 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/917665514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:21.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:21 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/917665514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:22.651 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:22 smithi171 ceph-mon[31139]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:22.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:22 smithi156 ceph-mon[39130]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:22.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:22 smithi156 ceph-mon[27570]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:24.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:42:24 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:42:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:42:24.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:24 smithi171 ceph-mon[31139]: pgmap v105: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:24.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:24 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2277957888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:24.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:24 smithi156 ceph-mon[27570]: pgmap v105: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:24.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:24 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2277957888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:24.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:24 smithi156 ceph-mon[39130]: pgmap v105: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:24.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:24 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2277957888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:26.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:26 smithi171 ceph-mon[31139]: pgmap v106: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:26.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:26 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3518065294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:26.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:42:26 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:42:26] "GET /metrics HTTP/1.1" 200 242046 "" "Prometheus/2.43.0" 2024-04-26T04:42:26.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:26 smithi156 ceph-mon[27570]: pgmap v106: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:26.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:26 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3518065294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:26.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:26 smithi156 ceph-mon[39130]: pgmap v106: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:26.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:26 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3518065294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:28.575 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:28 smithi156 ceph-mon[27570]: pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:28.575 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:28 smithi156 ceph-mon[39130]: pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:28.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:28 smithi171 ceph-mon[31139]: pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:29.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:29 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3810407317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:29.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:29 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:29.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:29 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:29.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:29 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:29.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:29 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:29.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:29 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3810407317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:29.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:29 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:29.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:29 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:29.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:29 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:29.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:29 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:29.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:29 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3810407317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:29.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:29 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:29 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:29 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:29.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:29 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:30.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:30 smithi171 ceph-mon[31139]: pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:30.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:30 smithi171 ceph-mon[31139]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (unknown) 2024-04-26T04:42:30.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:30 smithi171 ceph-mon[31139]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-26T04:42:30.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:30 smithi171 ceph-mon[31139]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-26T04:42:30.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:30 smithi156 ceph-mon[39130]: pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:30.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:30 smithi156 ceph-mon[39130]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (unknown) 2024-04-26T04:42:30.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:30 smithi156 ceph-mon[39130]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-26T04:42:30.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:30 smithi156 ceph-mon[39130]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-26T04:42:30.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:30 smithi156 ceph-mon[27570]: pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:30.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:30 smithi156 ceph-mon[27570]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (unknown) 2024-04-26T04:42:30.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:30 smithi156 ceph-mon[27570]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-26T04:42:30.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:30 smithi156 ceph-mon[27570]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-26T04:42:31.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:31 smithi171 ceph-mon[31139]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 on smithi171 2024-04-26T04:42:31.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:31 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3957731355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:31.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:31 smithi156 ceph-mon[27570]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 on smithi171 2024-04-26T04:42:31.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:31 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3957731355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:31.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:31 smithi156 ceph-mon[39130]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 on smithi171 2024-04-26T04:42:31.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:31 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3957731355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:32.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:32 smithi171 ceph-mon[31139]: pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:32 smithi156 ceph-mon[27570]: pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:32.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:32 smithi156 ceph-mon[39130]: pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:33.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:33 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3946767779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:33.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:33 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3946767779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:33.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:33 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3946767779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:34.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:34 smithi171 ceph-mon[31139]: pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:34.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:42:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:42:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:42:34.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:34 smithi156 ceph-mon[27570]: pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:34.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:34 smithi156 ceph-mon[39130]: pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:36.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:36 smithi171 ceph-mon[31139]: pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:36.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:36 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/570137040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:36.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:42:36 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:42:36] "GET /metrics HTTP/1.1" 200 239965 "" "Prometheus/2.43.0" 2024-04-26T04:42:36.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:36 smithi156 ceph-mon[27570]: pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:36.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:36 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/570137040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:36 smithi156 ceph-mon[39130]: pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:36 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/570137040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:38.645 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:38 smithi156 ceph-mon[27570]: pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:38.646 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:38 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1269339974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:38.646 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:38 smithi156 ceph-mon[39130]: pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:38.646 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:38 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1269339974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:38.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:38 smithi171 ceph-mon[31139]: pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:38.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:38 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1269339974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:40.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:40 smithi171 ceph-mon[31139]: pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:40.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:40 smithi156 ceph-mon[27570]: pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:40.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:40 smithi156 ceph-mon[39130]: pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:41.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:41 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3145493632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:41.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:41 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3145493632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:41.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:41 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3145493632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:42.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:42 smithi171 ceph-mon[31139]: pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:42.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:42 smithi156 ceph-mon[39130]: pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:42.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:42 smithi156 ceph-mon[27570]: pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:43.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:43 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/366922870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:43.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:43 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/366922870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:43.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:43 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/366922870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:44.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:44 smithi171 ceph-mon[31139]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:44.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:44 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:44.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:42:44 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:42:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:42:44.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:44 smithi156 ceph-mon[27570]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:44.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:44 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:44.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:44 smithi156 ceph-mon[39130]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:44.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:44 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:45.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:45 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/524218293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:45.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:45 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/524218293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:45.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:45 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/524218293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:46.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:46 smithi171 ceph-mon[31139]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:46.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:46 smithi156 ceph-mon[39130]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:46.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:46 smithi156 ceph-mon[27570]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:46.844 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:42:46 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:42:46] "GET /metrics HTTP/1.1" 200 239967 "" "Prometheus/2.43.0" 2024-04-26T04:42:48.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:48 smithi171 ceph-mon[31139]: pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:48.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:48 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3973406393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:48.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:48 smithi156 ceph-mon[39130]: pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:48.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:48 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3973406393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:48.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:48 smithi156 ceph-mon[27570]: pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:48.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:48 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3973406393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:50.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:50 smithi171 ceph-mon[31139]: pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:50.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:50 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/469756560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:50.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:50 smithi156 ceph-mon[39130]: pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:50.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:50 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/469756560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:50.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:50 smithi156 ceph-mon[27570]: pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:50.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:50 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/469756560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:52.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:52 smithi171 ceph-mon[31139]: pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:52.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:52 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2322670305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:52.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:52 smithi156 ceph-mon[27570]: pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:52.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:52 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2322670305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:52.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:52 smithi156 ceph-mon[39130]: pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:52.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:52 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2322670305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:54.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:54 smithi171 ceph-mon[31139]: pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:54.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:42:54 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[33240]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:42:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:42:54.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:54 smithi156 ceph-mon[27570]: pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:54.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:54 smithi156 ceph-mon[39130]: pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:55.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:55 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1454344474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:55.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:55 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1454344474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:55.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:55 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1454344474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:56.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:56 smithi171 ceph-mon[31139]: pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:56.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:56 smithi156 ceph-mon[27570]: pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:56.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:56 smithi156 ceph-mon[39130]: pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:56.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:42:56 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:42:56] "GET /metrics HTTP/1.1" 200 239967 "" "Prometheus/2.43.0" 2024-04-26T04:42:57.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:57 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3172097802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:57.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:57 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3172097802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:57.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:57 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3172097802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:58.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:58 smithi171 ceph-mon[31139]: pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:58.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:58 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:58.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:58 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:42:58.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:58 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:42:58.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:58 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:42:58.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:58 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:58.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:58 smithi156 ceph-mon[39130]: pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:58.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:58 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:58.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:58 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:42:58.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:58 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:42:58.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:58 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:42:58.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:58 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:58.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:58 smithi156 ceph-mon[27570]: pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:42:58.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:58 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:42:58.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:58 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:42:58.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:58 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:42:58.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:58 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:42:58.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:58 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:42:59.660 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:59 smithi171 ceph-mon[31139]: Upgrade: Updating mgr.x 2024-04-26T04:42:59.660 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:59 smithi171 ceph-mon[31139]: Deploying daemon mgr.x on smithi171 2024-04-26T04:42:59.660 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:42:59 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:59.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:59 smithi156 ceph-mon[27570]: Upgrade: Updating mgr.x 2024-04-26T04:42:59.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:59 smithi156 ceph-mon[27570]: Deploying daemon mgr.x on smithi171 2024-04-26T04:42:59.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:42:59 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:59.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:59 smithi156 ceph-mon[39130]: Upgrade: Updating mgr.x 2024-04-26T04:42:59.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:59 smithi156 ceph-mon[39130]: Deploying daemon mgr.x on smithi171 2024-04-26T04:42:59.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:42:59 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:00.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:42:59 smithi171 systemd[1]: Stopping Ceph mgr.x for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:43:00.414 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:00 smithi171 podman[66571]: 2024-04-26 04:43:00.153655769 +0000 UTC m=+0.137890950 container died 77edd00db79e1b66482692db88cb34e7aa8b704318102407cf7b54eb6c7626b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-26T04:43:00.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:00 smithi171 ceph-mon[31139]: pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:00.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:00 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3304003867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:00.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:00 smithi171 podman[66571]: 2024-04-26 04:43:00.505438945 +0000 UTC m=+0.489674134 container cleanup 77edd00db79e1b66482692db88cb34e7aa8b704318102407cf7b54eb6c7626b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD) 2024-04-26T04:43:00.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:00 smithi171 bash[66571]: ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x 2024-04-26T04:43:00.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:00 smithi156 ceph-mon[27570]: pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:00.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:00 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3304003867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:00.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:00 smithi156 ceph-mon[39130]: pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:00.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:00 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3304003867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:01.131 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:00 smithi171 podman[66591]: 2024-04-26 04:43:00.782343717 +0000 UTC m=+0.623606853 container remove 77edd00db79e1b66482692db88cb34e7aa8b704318102407cf7b54eb6c7626b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=) 2024-04-26T04:43:01.131 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:00 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:43:01.131 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:00 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-26T04:43:01.131 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:00 smithi171 systemd[1]: Stopped Ceph mgr.x for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:43:01.131 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:00 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.x.service: Consumed 17.319s CPU time. 2024-04-26T04:43:01.465 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:01 smithi171 systemd[1]: Starting Ceph mgr.x for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:43:01.465 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:01 smithi171 podman[66729]: 2024-04-26 04:43:01.246304997 +0000 UTC m=+0.017739988 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:43:01.466 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:01 smithi171 podman[66729]: 2024-04-26 04:43:01.35036104 +0000 UTC m=+0.121796064 container create 434a72ecc6a494c1da2650dac79c92e0f4f8031f49ac5ab7ac82037f03b2b0dc (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-26T04:43:01.986 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:01 smithi171 podman[66729]: 2024-04-26 04:43:01.630977736 +0000 UTC m=+0.402412714 container init 434a72ecc6a494c1da2650dac79c92e0f4f8031f49ac5ab7ac82037f03b2b0dc (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb) 2024-04-26T04:43:01.986 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:01 smithi171 podman[66729]: 2024-04-26 04:43:01.636130959 +0000 UTC m=+0.407565933 container start 434a72ecc6a494c1da2650dac79c92e0f4f8031f49ac5ab7ac82037f03b2b0dc (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-26T04:43:01.986 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:01 smithi171 ceph-mgr[66767]: -- 172.21.15.171:0/3260527028 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56074095a5a0 con 0x56074092f000 2024-04-26T04:43:01.986 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:01 smithi171 bash[66729]: 434a72ecc6a494c1da2650dac79c92e0f4f8031f49ac5ab7ac82037f03b2b0dc 2024-04-26T04:43:01.986 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:01 smithi171 systemd[1]: Started Ceph mgr.x for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:43:01.986 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:01.790+0000 7ff234ea2200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:43:01.986 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:01.985+0000 7ff234ea2200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:43:02.249 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:02 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:02.099+0000 7ff234ea2200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:43:02.668 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:02 smithi156 ceph-mon[27570]: pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:02.668 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:02 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:02.668 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:02 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:02.669 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:02 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:02.669 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:02 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2204301589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:02.669 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:02 smithi156 ceph-mon[39130]: pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:02.669 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:02 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:02.669 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:02 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:02.669 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:02 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:02.670 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:02 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2204301589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:02.671 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:02 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:02.429+0000 7ff234ea2200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:43:02.671 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:02 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:02.520+0000 7ff234ea2200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:43:02.671 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:02 smithi171 ceph-mon[31139]: pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:02.672 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:02 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:02.672 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:02 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:02.672 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:02 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:02.672 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:02 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2204301589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:04.145 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:03.767+0000 7ff234ea2200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:43:04.145 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:03.970+0000 7ff234ea2200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:43:04.145 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:04 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:04.144+0000 7ff234ea2200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:43:04.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:04 smithi156 ceph-mon[27570]: pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:04.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:04 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2468222899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:04.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:04 smithi156 ceph-mon[39130]: pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:04.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:04 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2468222899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:04.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:04 smithi171 ceph-mon[31139]: pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:04.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:04 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2468222899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:04.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:04 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:04.445+0000 7ff234ea2200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:43:04.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:04 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:04.540+0000 7ff234ea2200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:43:05.204 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:04 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:04.755+0000 7ff234ea2200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:43:05.204 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:04 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:04.953+0000 7ff234ea2200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:43:05.974 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:05 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:05.670+0000 7ff234ea2200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:43:05.977 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:05 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:05.977 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:05 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:05.977 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:05 smithi171 ceph-mon[31139]: pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:06.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:05 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:06.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:05 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:06.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:05 smithi156 ceph-mon[27570]: pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:06.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:05 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:06.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:05 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:06.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:05 smithi156 ceph-mon[39130]: pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:06.249 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:06 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:06.108+0000 7ff234ea2200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:43:06.249 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:06 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:06.207+0000 7ff234ea2200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:43:06.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:06 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:06.453+0000 7ff234ea2200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:43:06.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:06 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:43:06] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2024-04-26T04:43:07.375 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:07 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:07.012+0000 7ff234ea2200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:43:07.375 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:07 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:07.113+0000 7ff234ea2200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:43:07.375 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:07 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:07.373+0000 7ff234ea2200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:43:07.628 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:07 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:07.517+0000 7ff234ea2200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:43:07.629 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:07 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:07.626+0000 7ff234ea2200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:43:07.904 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:07 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:07.752+0000 7ff234ea2200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:43:07.905 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:07 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:07.847+0000 7ff234ea2200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:43:07.905 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:07 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:07.905 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:07 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:07.905 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:07 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/802960267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:07.905 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:07 smithi171 ceph-mon[31139]: pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:07.905 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:07 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:07.905 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:07 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:08.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:07 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:08.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:07 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:08.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:07 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/802960267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:08.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:07 smithi156 ceph-mon[39130]: pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:08.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:07 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:08.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:07 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:08.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:07 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:08.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:07 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:08.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:07 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/802960267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:08.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:07 smithi156 ceph-mon[27570]: pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:08.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:07 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:08.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:07 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:08.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:08 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:08.693+0000 7ff234ea2200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:43:09.137 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:08 smithi171 ceph-mon[31139]: Standby manager daemon x restarted 2024-04-26T04:43:09.137 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:08 smithi171 ceph-mon[31139]: Standby manager daemon x started 2024-04-26T04:43:09.137 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:08 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:43:09.137 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:08 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:43:09.138 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:08 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:43:09.138 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:08 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:43:09.138 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:08 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:08.918+0000 7ff234ea2200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:43:09.138 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:08 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:43:08] ENGINE Bus STARTING 2024-04-26T04:43:09.138 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:09 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: CherryPy Checker: 2024-04-26T04:43:09.138 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:09 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: The Application mounted at '' has an empty config. 2024-04-26T04:43:09.138 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:09 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:09.138 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:09 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:43:09] ENGINE Serving on http://:::9283 2024-04-26T04:43:09.138 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:09 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:43:09] ENGINE Bus STARTED 2024-04-26T04:43:09.254 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:08 smithi156 ceph-mon[39130]: Standby manager daemon x restarted 2024-04-26T04:43:09.254 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:08 smithi156 ceph-mon[39130]: Standby manager daemon x started 2024-04-26T04:43:09.254 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:08 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:43:09.254 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:08 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:43:09.255 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:08 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:43:09.255 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:08 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:43:09.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:08 smithi156 ceph-mon[27570]: Standby manager daemon x restarted 2024-04-26T04:43:09.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:08 smithi156 ceph-mon[27570]: Standby manager daemon x started 2024-04-26T04:43:09.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:08 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:43:09.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:08 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:43:09.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:08 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:43:09.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:08 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:43:10.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:09 smithi171 ceph-mon[31139]: mgrmap e21: y(active, since 3m), standbys: x 2024-04-26T04:43:10.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:09 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1368733537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:10.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:09 smithi171 ceph-mon[31139]: pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:10.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:09 smithi156 ceph-mon[27570]: mgrmap e21: y(active, since 3m), standbys: x 2024-04-26T04:43:10.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:09 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1368733537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:10.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:09 smithi156 ceph-mon[27570]: pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:10.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:09 smithi156 ceph-mon[39130]: mgrmap e21: y(active, since 3m), standbys: x 2024-04-26T04:43:10.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:09 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1368733537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:10.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:09 smithi156 ceph-mon[39130]: pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:10.993 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:10 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:43:10] ENGINE Bus STOPPING 2024-04-26T04:43:10.993 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:10 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:43:10] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:43:10.993 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:10 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:43:10] ENGINE Bus STOPPED 2024-04-26T04:43:11.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:43:11] ENGINE Bus STARTING 2024-04-26T04:43:11.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:11.092+0000 7ff192a8d700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:11.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:11.092+0000 7ff192a8d700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:11.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:11.092+0000 7ff192a8d700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:11.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:11.092+0000 7ff192a8d700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:11.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:11.092+0000 7ff192a8d700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:11.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:11.096+0000 7ff196294700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:11.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:11.096+0000 7ff196294700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:11.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:11.096+0000 7ff196294700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:11.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:11.096+0000 7ff196294700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:11.251 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:43:11.096+0000 7ff196294700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:11.251 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:43:11] ENGINE Serving on http://:::9283 2024-04-26T04:43:11.251 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:43:11] ENGINE Bus STARTED 2024-04-26T04:43:11.308 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:10 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:10.965+0000 7fac15051700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-26T04:43:11.308 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ignoring --setuser ceph since I am not root 2024-04-26T04:43:11.308 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ignoring --setgroup ceph since I am not root 2024-04-26T04:43:11.308 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mgr[27844]: -- 172.21.15.156:0/897231011 <== mon.0 v2:172.21.15.156:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5605ec7049c0 con 0x5605ec6da800 2024-04-26T04:43:11.308 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:11.132+0000 7fe83917c1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:43:11.308 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:11.306+0000 7fe83917c1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:43:11.593 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:11.413+0000 7fe83917c1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:43:12.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:11.733+0000 7fe83917c1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:43:12.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:11.817+0000 7fe83917c1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:43:12.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:12.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:12.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:12.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:12.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:12.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:12.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:43:12.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:43:12.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: Activating manager daemon x 2024-04-26T04:43:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T04:43:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-26T04:43:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: mgrmap e22: x(active, starting, since 0.00635939s) 2024-04-26T04:43:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:43:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:43:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:43:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.wyfzii"}]: dispatch 2024-04-26T04:43:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.kzjlqn"}]: dispatch 2024-04-26T04:43:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:43:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:43:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:43:12.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:43:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:43:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:43:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:43:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:43:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:43:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:43:12.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: Manager daemon x is now available 2024-04-26T04:43:12.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:12.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:12.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:12.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:12.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:12.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:12.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:43:12.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:43:12.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: Activating manager daemon x 2024-04-26T04:43:12.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T04:43:12.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-26T04:43:12.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: mgrmap e22: x(active, starting, since 0.00635939s) 2024-04-26T04:43:12.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:43:12.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:43:12.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:43:12.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.wyfzii"}]: dispatch 2024-04-26T04:43:12.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.kzjlqn"}]: dispatch 2024-04-26T04:43:12.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:43:12.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:43:12.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:43:12.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:43:12.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:43:12.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:43:12.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:43:12.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:12.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:12.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:43:12.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:43:12.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:43:12.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: Manager daemon x is now available 2024-04-26T04:43:12.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:12.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:12.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:43:12.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:43:12.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:43:12.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:43:12.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: Deploying cephadm binary to smithi171 2024-04-26T04:43:12.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: Deploying cephadm binary to smithi156 2024-04-26T04:43:12.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/799349287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:12.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:12.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:12.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:43:12.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:43:12.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:43:12.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:43:12.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: Deploying cephadm binary to smithi171 2024-04-26T04:43:12.101 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: Deploying cephadm binary to smithi156 2024-04-26T04:43:12.101 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:11 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/799349287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:12.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:12.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:12.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:12.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:12.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' 2024-04-26T04:43:12.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:12.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24344 172.21.15.156:0/2391072195' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:43:12.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:43:12.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: Activating manager daemon x 2024-04-26T04:43:12.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T04:43:12.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24344 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-26T04:43:12.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: mgrmap e22: x(active, starting, since 0.00635939s) 2024-04-26T04:43:12.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:43:12.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:43:12.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:43:12.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.wyfzii"}]: dispatch 2024-04-26T04:43:12.252 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.kzjlqn"}]: dispatch 2024-04-26T04:43:12.252 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:43:12.252 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:43:12.252 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:43:12.252 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:43:12.252 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:43:12.252 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:43:12.252 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:43:12.252 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:12.252 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:12.252 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:43:12.252 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:43:12.252 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:43:12.252 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: Manager daemon x is now available 2024-04-26T04:43:12.253 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:12.253 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:12.253 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:43:12.253 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:43:12.253 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:43:12.253 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:43:12.253 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: Deploying cephadm binary to smithi171 2024-04-26T04:43:12.253 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: Deploying cephadm binary to smithi156 2024-04-26T04:43:12.253 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:11 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/799349287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:13.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:12 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:13.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:12 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:13.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:12 smithi171 ceph-mon[31139]: mgrmap e23: x(active, since 1.01321s) 2024-04-26T04:43:13.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:12 smithi171 ceph-mon[31139]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:13.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:12 smithi171 ceph-mon[31139]: [26/Apr/2024:04:43:12] ENGINE Bus STARTING 2024-04-26T04:43:13.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:12 smithi171 ceph-mon[31139]: [26/Apr/2024:04:43:12] ENGINE Serving on https://172.21.15.171:7150 2024-04-26T04:43:13.326 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:12 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:12.987+0000 7fe83917c1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:43:13.327 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:13 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:13.155+0000 7fe83917c1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:43:13.327 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:12 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:13.327 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:12 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:13.327 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:12 smithi156 ceph-mon[27570]: mgrmap e23: x(active, since 1.01321s) 2024-04-26T04:43:13.327 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:12 smithi156 ceph-mon[27570]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:13.327 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:12 smithi156 ceph-mon[27570]: [26/Apr/2024:04:43:12] ENGINE Bus STARTING 2024-04-26T04:43:13.327 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:12 smithi156 ceph-mon[27570]: [26/Apr/2024:04:43:12] ENGINE Serving on https://172.21.15.171:7150 2024-04-26T04:43:13.328 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:12 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:13.328 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:12 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:13.328 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:12 smithi156 ceph-mon[39130]: mgrmap e23: x(active, since 1.01321s) 2024-04-26T04:43:13.328 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:12 smithi156 ceph-mon[39130]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:13.328 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:12 smithi156 ceph-mon[39130]: [26/Apr/2024:04:43:12] ENGINE Bus STARTING 2024-04-26T04:43:13.328 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:12 smithi156 ceph-mon[39130]: [26/Apr/2024:04:43:12] ENGINE Serving on https://172.21.15.171:7150 2024-04-26T04:43:13.593 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:13 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:13.325+0000 7fe83917c1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:43:13.973 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:13 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:13.596+0000 7fe83917c1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:43:13.973 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:13 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:13.691+0000 7fe83917c1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:43:13.973 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:13 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:13.786+0000 7fe83917c1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:43:14.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:13 smithi171 ceph-mon[31139]: [26/Apr/2024:04:43:12] ENGINE Serving on http://172.21.15.171:8765 2024-04-26T04:43:14.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:13 smithi171 ceph-mon[31139]: [26/Apr/2024:04:43:12] ENGINE Bus STARTED 2024-04-26T04:43:14.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:13 smithi171 ceph-mon[31139]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:14.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:13 smithi171 ceph-mon[31139]: mgrmap e24: x(active, since 2s) 2024-04-26T04:43:14.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:13 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2615069496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:14.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:13 smithi156 ceph-mon[27570]: [26/Apr/2024:04:43:12] ENGINE Serving on http://172.21.15.171:8765 2024-04-26T04:43:14.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:13 smithi156 ceph-mon[27570]: [26/Apr/2024:04:43:12] ENGINE Bus STARTED 2024-04-26T04:43:14.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:13 smithi156 ceph-mon[27570]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:14.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:13 smithi156 ceph-mon[27570]: mgrmap e24: x(active, since 2s) 2024-04-26T04:43:14.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:13 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2615069496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:14.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:14.073+0000 7fe83917c1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:43:14.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:13 smithi156 ceph-mon[39130]: [26/Apr/2024:04:43:12] ENGINE Serving on http://172.21.15.171:8765 2024-04-26T04:43:14.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:13 smithi156 ceph-mon[39130]: [26/Apr/2024:04:43:12] ENGINE Bus STARTED 2024-04-26T04:43:14.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:13 smithi156 ceph-mon[39130]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:14.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:13 smithi156 ceph-mon[39130]: mgrmap e24: x(active, since 2s) 2024-04-26T04:43:14.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:13 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2615069496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:14.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:14 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:43:14] "GET /metrics HTTP/1.1" 200 30875 "" "Prometheus/2.43.0" 2024-04-26T04:43:14.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:14.536+0000 7fe83917c1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:43:15.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:15.068+0000 7fe83917c1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:43:15.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:15.163+0000 7fe83917c1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:43:15.641 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:15.344+0000 7fe83917c1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:43:15.900 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:15.639+0000 7fe83917c1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:43:16.155 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:15 smithi156 ceph-mon[27570]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:16.155 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:15 smithi156 ceph-mon[27570]: mgrmap e25: x(active, since 4s) 2024-04-26T04:43:16.156 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:15 smithi156 ceph-mon[39130]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:16.156 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:15 smithi156 ceph-mon[39130]: mgrmap e25: x(active, since 4s) 2024-04-26T04:43:16.156 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:15.898+0000 7fe83917c1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:43:16.156 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:16.153+0000 7fe83917c1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:43:16.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:15 smithi171 ceph-mon[31139]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:16.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:15 smithi171 ceph-mon[31139]: mgrmap e25: x(active, since 4s) 2024-04-26T04:43:16.406 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:16.302+0000 7fe83917c1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:43:16.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:16.404+0000 7fe83917c1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:43:16.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:16.527+0000 7fe83917c1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:43:16.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:16.627+0000 7fe83917c1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:43:17.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:16 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/421662974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:17.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:16 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/421662974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:17.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:16 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/421662974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:17.797 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:17.468+0000 7fe83917c1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:43:17.797 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:17.687+0000 7fe83917c1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:43:17.798 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:43:17] ENGINE Bus STARTING 2024-04-26T04:43:17.798 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: CherryPy Checker: 2024-04-26T04:43:17.798 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: The Application mounted at '' has an empty config. 2024-04-26T04:43:17.798 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: 2024-04-26T04:43:18.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-mon[39130]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:18.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-mon[39130]: Standby manager daemon y started 2024-04-26T04:43:18.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.156:0/1112891272' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:43:18.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.156:0/1112891272' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:43:18.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.156:0/1112891272' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:43:18.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.156:0/1112891272' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:43:18.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-mon[27570]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:18.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-mon[27570]: Standby manager daemon y started 2024-04-26T04:43:18.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.156:0/1112891272' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:43:18.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.156:0/1112891272' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:43:18.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.156:0/1112891272' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:43:18.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.156:0/1112891272' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:43:18.095 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:43:17] ENGINE Serving on http://:::9283 2024-04-26T04:43:18.095 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: [26/Apr/2024:04:43:17] ENGINE Bus STARTED 2024-04-26T04:43:18.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:17 smithi171 ceph-mon[31139]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:18.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:17 smithi171 ceph-mon[31139]: Standby manager daemon y started 2024-04-26T04:43:18.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:17 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.156:0/1112891272' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:43:18.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:17 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.156:0/1112891272' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:43:18.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:17 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.156:0/1112891272' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:43:18.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:17 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.156:0/1112891272' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:43:19.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:18 smithi171 ceph-mon[31139]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-26T04:43:19.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:18 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:43:19.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:18 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1978403085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:19.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:18 smithi156 ceph-mon[27570]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-26T04:43:19.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:18 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:43:19.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:18 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1978403085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:19.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:18 smithi156 ceph-mon[39130]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-26T04:43:19.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:18 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:43:19.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:18 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1978403085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:20.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:20 smithi156 ceph-mon[39130]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:43:20.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:20 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:20.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:20 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:20.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:20 smithi156 ceph-mon[27570]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:43:20.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:20 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:20.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:20 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:20.609 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:20 smithi171 ceph-mon[31139]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:43:20.609 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:20 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:20.609 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:20 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:21.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:21 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:21.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:21 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:21.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:21 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:21 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:21 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:21 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:21 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:21 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:21 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:21 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:21 smithi171 ceph-mon[31139]: Adjusting osd_memory_target on smithi156 to 2480M 2024-04-26T04:43:21.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:21 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:21.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:21 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:21.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:21 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:21.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:21 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3237543063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:21.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:21.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:21.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[27570]: Adjusting osd_memory_target on smithi156 to 2480M 2024-04-26T04:43:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:21.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3237543063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:21.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:21.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:21.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:21.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[39130]: Adjusting osd_memory_target on smithi156 to 2480M 2024-04-26T04:43:21.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:21.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:21.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:21.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:21 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3237543063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:22.652 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:22 smithi171 ceph-mon[31139]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:43:22.653 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:22 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:22.653 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:22 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:22.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:22 smithi156 ceph-mon[27570]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:43:22.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:22 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:22.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:22 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:22.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:22 smithi156 ceph-mon[39130]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:43:22.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:22 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:22.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:22 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:24.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:24 smithi171 ceph-mon[31139]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T04:43:24.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:24 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1338010863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:24.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:24 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:43:24] "GET /metrics HTTP/1.1" 200 30875 "" "Prometheus/2.43.0" 2024-04-26T04:43:24.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:24 smithi156 ceph-mon[27570]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T04:43:24.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:24 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1338010863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:24.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:24 smithi156 ceph-mon[39130]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T04:43:24.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:24 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1338010863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:26.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:26 smithi171 ceph-mon[31139]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:43:26.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:26 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/785222009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:26.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:26 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:26.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:26 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:43:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:43:26.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:26 smithi156 ceph-mon[27570]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:43:26.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:26 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/785222009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:26.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:26 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:26.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:26 smithi156 ceph-mon[39130]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:43:26.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:26 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/785222009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:26.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:26 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:28.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:28 smithi171 ceph-mon[31139]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:43:28.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:28 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3326888866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:28.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:28 smithi156 ceph-mon[27570]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:43:28.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:28 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3326888866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:28.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:28 smithi156 ceph-mon[39130]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:43:28.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:28 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3326888866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:30.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:30 smithi171 ceph-mon[31139]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:43:30.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:30 smithi156 ceph-mon[27570]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:43:30.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:30 smithi156 ceph-mon[39130]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:43:31.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:31 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1261873309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:31.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:31 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1261873309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:31.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:31 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1261873309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:32.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:32 smithi171 ceph-mon[31139]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:32 smithi156 ceph-mon[27570]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:32.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:32 smithi156 ceph-mon[39130]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:33.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:33 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2646905051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:33.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:33 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2646905051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:33.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:33 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2646905051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:34.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:34 smithi171 ceph-mon[31139]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:34.500 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:43:34] "GET /metrics HTTP/1.1" 200 33721 "" "Prometheus/2.43.0" 2024-04-26T04:43:34.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:34 smithi156 ceph-mon[27570]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:34.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:34 smithi156 ceph-mon[39130]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:36.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:36 smithi171 ceph-mon[31139]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:36.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:36 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/386519728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:36.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:36 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:36.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:36 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:36.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:36 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:36 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:36 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:36 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:36 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:36 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:36 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:36 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:36 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:36.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:36 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:36.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:36 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:36.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:43:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:43:36.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[27570]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:36.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/386519728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:36.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:36.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:36.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:36.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:36.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:36.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[39130]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:36.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/386519728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:36.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:36.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:36.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:36.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:36.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:36.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:36 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: Adjusting osd_memory_target on smithi171 to 2480M 2024-04-26T04:43:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: Updating smithi156:/etc/ceph/ceph.conf 2024-04-26T04:43:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: Updating smithi171:/etc/ceph/ceph.conf 2024-04-26T04:43:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:43:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:43:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:43:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:43:37.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:43:37.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:43:37.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:37 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.792 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: Adjusting osd_memory_target on smithi171 to 2480M 2024-04-26T04:43:37.792 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: Updating smithi156:/etc/ceph/ceph.conf 2024-04-26T04:43:37.792 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: Updating smithi171:/etc/ceph/ceph.conf 2024-04-26T04:43:37.793 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:43:37.793 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:43:37.793 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:43:37.793 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:43:37.793 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:43:37.793 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:43:37.793 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.793 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.793 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.793 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.793 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.793 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.793 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.793 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.793 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.794 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: Adjusting osd_memory_target on smithi171 to 2480M 2024-04-26T04:43:37.794 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: Updating smithi156:/etc/ceph/ceph.conf 2024-04-26T04:43:37.794 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: Updating smithi171:/etc/ceph/ceph.conf 2024-04-26T04:43:37.794 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:43:37.794 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:43:37.794 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:43:37.794 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:43:37.794 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:43:37.794 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:43:37.794 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.794 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.795 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.795 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.795 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.795 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.795 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.795 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:37.795 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:37 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:38.664 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:38 smithi156 ceph-mon[27570]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:43:38.664 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:38 smithi156 ceph-mon[27570]: Reconfiguring daemon alertmanager.a on smithi156 2024-04-26T04:43:38.664 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:38 smithi156 ceph-mon[27570]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:38.664 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:38 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1955184415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:38.664 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:38 smithi156 ceph-mon[39130]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:43:38.664 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:38 smithi156 ceph-mon[39130]: Reconfiguring daemon alertmanager.a on smithi156 2024-04-26T04:43:38.664 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:38 smithi156 ceph-mon[39130]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:38.665 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:38 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1955184415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:38.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:38 smithi171 ceph-mon[31139]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:43:38.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:38 smithi171 ceph-mon[31139]: Reconfiguring daemon alertmanager.a on smithi156 2024-04-26T04:43:38.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:38 smithi171 ceph-mon[31139]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:38.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:38 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1955184415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:40.403 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:40 smithi156 systemd[1]: Stopping Ceph alertmanager.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:43:40.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:40 smithi171 ceph-mon[31139]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:40.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:40 smithi156 ceph-mon[27570]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:40.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:40 smithi156 ceph-mon[39130]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:40.844 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:40 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[75609]: ts=2024-04-26T04:43:40.595Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:43:40.844 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:40 smithi156 podman[88571]: 2024-04-26 04:43:40.636293114 +0000 UTC m=+0.251251289 container died 9742e32fd5fc5725890b5f85bf5498c37aff42d5c948228490cfb7a06b29a091 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:41.319 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:40 smithi156 podman[88571]: 2024-04-26 04:43:40.984190363 +0000 UTC m=+0.599148536 container cleanup 9742e32fd5fc5725890b5f85bf5498c37aff42d5c948228490cfb7a06b29a091 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:41.319 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:40 smithi156 bash[88571]: ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T04:43:41.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:41 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1660934538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:41.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:41 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:41.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:41 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1660934538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:41.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:41 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:41.594 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:41 smithi156 podman[88593]: 2024-04-26 04:43:41.318830472 +0000 UTC m=+0.680980440 container remove 9742e32fd5fc5725890b5f85bf5498c37aff42d5c948228490cfb7a06b29a091 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:41.594 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:41 smithi156 podman[88593]: 2024-04-26 04:43:41.411491774 +0000 UTC m=+0.773641700 volume remove 15e029a4af625eede17cf0e28f932443c408a6ccbe722d2a6b8f74bcd08210c0 2024-04-26T04:43:41.594 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:41 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T04:43:41.594 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:41 smithi156 systemd[1]: Stopped Ceph alertmanager.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:43:41.594 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:41 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.177s CPU time. 2024-04-26T04:43:41.594 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:41 smithi156 systemd[1]: Starting Ceph alertmanager.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:43:41.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:41 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1660934538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:41.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:41 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:42.079 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:41 smithi156 podman[88691]: 2024-04-26 04:43:41.673862969 +0000 UTC m=+0.022516555 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-26T04:43:42.079 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:41 smithi156 podman[88691]: 2024-04-26 04:43:41.78777888 +0000 UTC m=+0.136432510 volume create 7ef5b1e73ba0732ba8f2f2bad80d9d55e39038796eaabcdfd5d7858275d0dc3d 2024-04-26T04:43:42.079 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:41 smithi156 podman[88691]: 2024-04-26 04:43:41.896446629 +0000 UTC m=+0.245100214 container create 932345b7da326190269cf5d4e04632a5b73e593424331c04cd0b3df2dad8b12e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:42.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:42 smithi156 podman[88691]: 2024-04-26 04:43:42.174108519 +0000 UTC m=+0.522762103 container init 932345b7da326190269cf5d4e04632a5b73e593424331c04cd0b3df2dad8b12e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:42.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:42 smithi156 podman[88691]: 2024-04-26 04:43:42.179746013 +0000 UTC m=+0.528399599 container start 932345b7da326190269cf5d4e04632a5b73e593424331c04cd0b3df2dad8b12e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:42.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:43:42.191Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-26T04:43:42.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:43:42.191Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-26T04:43:42.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:43:42.193Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.156 port=9094 2024-04-26T04:43:42.594 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:43:42.194Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T04:43:42.594 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:43:42.217Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:43:42.594 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:43:42.217Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:43:42.594 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:43:42.220Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-26T04:43:42.594 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:43:42.220Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-26T04:43:42.594 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:42 smithi156 bash[88691]: 932345b7da326190269cf5d4e04632a5b73e593424331c04cd0b3df2dad8b12e 2024-04-26T04:43:42.594 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:42 smithi156 systemd[1]: Started Ceph alertmanager.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:43:42.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:42 smithi171 ceph-mon[31139]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:43.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:42 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:43.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:42 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:43.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:42 smithi156 ceph-mon[39130]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:43.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:42 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:43.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:42 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:43.094 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:42 smithi156 systemd[1]: Stopping Ceph node-exporter.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:43:43.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:42 smithi156 ceph-mon[27570]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:43.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:42 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:43.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:42 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:43.404 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:43 smithi156 podman[88885]: 2024-04-26 04:43:43.140863156 +0000 UTC m=+0.126531696 container died 71daaa840a85f2b13d03559d7a2873cbd0a533b3d3f453c522cf55cc5ee67449 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:43.660 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:43 smithi156 podman[88885]: 2024-04-26 04:43:43.502406938 +0000 UTC m=+0.488075471 container cleanup 71daaa840a85f2b13d03559d7a2873cbd0a533b3d3f453c522cf55cc5ee67449 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:43.660 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:43 smithi156 bash[88885]: ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a 2024-04-26T04:43:43.914 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:43 smithi156 podman[88904]: 2024-04-26 04:43:43.812125916 +0000 UTC m=+0.676249339 container remove 71daaa840a85f2b13d03559d7a2873cbd0a533b3d3f453c522cf55cc5ee67449 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:43.914 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:43 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:43:43.914 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:43 smithi156 ceph-mon[27570]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T04:43:43.915 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:43 smithi156 ceph-mon[27570]: Reconfiguring daemon node-exporter.a on smithi156 2024-04-26T04:43:43.915 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:43 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2842210312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:43.915 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:43 smithi156 ceph-mon[39130]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T04:43:43.915 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:43 smithi156 ceph-mon[39130]: Reconfiguring daemon node-exporter.a on smithi156 2024-04-26T04:43:43.915 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:43 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2842210312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:43.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:43 smithi171 ceph-mon[31139]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T04:43:44.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:43 smithi171 ceph-mon[31139]: Reconfiguring daemon node-exporter.a on smithi156 2024-04-26T04:43:44.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:43 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2842210312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:44.195 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:43 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-26T04:43:44.195 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:43 smithi156 systemd[1]: Stopped Ceph node-exporter.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:43:44.195 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:43 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 2.244s CPU time. 2024-04-26T04:43:44.195 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:43 smithi156 systemd[1]: Starting Ceph node-exporter.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:43:44.195 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 podman[89002]: 2024-04-26 04:43:44.042670567 +0000 UTC m=+0.017430790 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:43:44.195 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 podman[89002]: 2024-04-26 04:43:44.171631508 +0000 UTC m=+0.146391776 container create 5af6998cc7e3ec4f95304ee91bae3375eceea078bc6b4e5886b9749459a5d1b7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:44.498 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:43:44.194Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00039438s 2024-04-26T04:43:44.499 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 podman[89002]: 2024-04-26 04:43:44.407670591 +0000 UTC m=+0.382430824 container init 5af6998cc7e3ec4f95304ee91bae3375eceea078bc6b4e5886b9749459a5d1b7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:44.499 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 podman[89002]: 2024-04-26 04:43:44.413575315 +0000 UTC m=+0.388335541 container start 5af6998cc7e3ec4f95304ee91bae3375eceea078bc6b4e5886b9749459a5d1b7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:44.499 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.416Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:43:44.499 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.416Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:43:44.499 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:43:44.499 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:43:44.499 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:43:44.499 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:43:44.500 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:43:44.500 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:43:44.500 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:43:44.500 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:43:44.500 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:43:44.500 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:43:44.500 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:43:44.500 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:43:44.501 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:43:44.501 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:43:44.501 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:43:44.501 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:43:44.501 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:43:44.501 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:43:44.501 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:43:44.501 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:43:44.501 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:43:44.501 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:43:44.501 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:43:44.501 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:43:44.502 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:43:44.502 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:43:44.502 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:43:44.502 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:43:44.502 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:43:44.502 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:43:44.502 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:43:44.502 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:43:44.502 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.417Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:43:44.502 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:43:44.502 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:43:44.502 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:43:44.502 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:43:44.503 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:43:44.503 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:43:44.503 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:43:44.503 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:43:44.503 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:43:44.503 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:43:44.503 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:43:44.503 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:43:44.503 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:43:44.503 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:43:44.503 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:43:44.503 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:43:44.503 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:43:44.504 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a[89016]: ts=2024-04-26T04:43:44.418Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:43:44.504 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 bash[89002]: 5af6998cc7e3ec4f95304ee91bae3375eceea078bc6b4e5886b9749459a5d1b7 2024-04-26T04:43:44.665 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:43:44 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:43:44] "GET /metrics HTTP/1.1" 200 33717 "" "Prometheus/2.43.0" 2024-04-26T04:43:44.843 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:43:44 smithi156 systemd[1]: Started Ceph node-exporter.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:43:44.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-mon[39130]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:44.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:44.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:44.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-mon[27570]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:44.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:44.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:44 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:44.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:44 smithi171 ceph-mon[31139]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:44.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:44 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:44.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:44 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:45.858 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:45 smithi171 ceph-mon[31139]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:43:45.859 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:45 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/471902413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:46.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:45 smithi156 ceph-mon[27570]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:43:46.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:45 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/471902413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:46.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:45 smithi156 ceph-mon[39130]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:43:46.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:45 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/471902413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:46.667 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:43:46 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[27821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:43:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:43:46.891 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:46 smithi171 ceph-mon[31139]: Reconfiguring daemon prometheus.a on smithi171 2024-04-26T04:43:46.891 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:46 smithi171 ceph-mon[31139]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:47.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:46 smithi156 ceph-mon[27570]: Reconfiguring daemon prometheus.a on smithi171 2024-04-26T04:43:47.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:46 smithi156 ceph-mon[27570]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:47.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:46 smithi156 ceph-mon[39130]: Reconfiguring daemon prometheus.a on smithi171 2024-04-26T04:43:47.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:46 smithi156 ceph-mon[39130]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:48.432 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:48 smithi171 systemd[1]: Stopping Ceph prometheus.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:43:48.730 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:48 smithi171 ceph-mon[31139]: pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:48.730 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:48 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2958581158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:48.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:48 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:43:48.431Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:43:48.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:48 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:43:48.431Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T04:43:48.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:48 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:43:48.431Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T04:43:48.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:48 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:43:48.431Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T04:43:48.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:48 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:43:48.431Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T04:43:48.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:48 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:43:48.431Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T04:43:48.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:48 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:43:48.431Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T04:43:48.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:48 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:43:48.431Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T04:43:48.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:48 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:43:48.431Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T04:43:48.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:48 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:43:48.432Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T04:43:48.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:48 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:43:48.432Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T04:43:48.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:48 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[63662]: ts=2024-04-26T04:43:48.432Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T04:43:48.732 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:48 smithi171 podman[75445]: 2024-04-26 04:43:48.441744646 +0000 UTC m=+0.115575524 container died be422f8a7019247d962c70f402adf51675e40b2fa041bffa95920a63989de9c1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:48.999 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:48 smithi171 podman[75445]: 2024-04-26 04:43:48.828970563 +0000 UTC m=+0.502801440 container cleanup be422f8a7019247d962c70f402adf51675e40b2fa041bffa95920a63989de9c1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:49.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:48 smithi171 bash[75445]: ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T04:43:49.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:48 smithi156 ceph-mon[39130]: pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:49.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:48 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2958581158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:49.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:48 smithi156 ceph-mon[27570]: pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:49.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:48 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2958581158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:49.463 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 podman[75464]: 2024-04-26 04:43:49.114454002 +0000 UTC m=+0.674606760 container remove be422f8a7019247d962c70f402adf51675e40b2fa041bffa95920a63989de9c1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:49.463 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T04:43:49.463 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 systemd[1]: Stopped Ceph prometheus.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:43:49.463 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.680s CPU time. 2024-04-26T04:43:49.463 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 systemd[1]: Starting Ceph prometheus.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:43:49.722 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 podman[75570]: 2024-04-26 04:43:49.367064273 +0000 UTC m=+0.019458677 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T04:43:49.722 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 podman[75570]: 2024-04-26 04:43:49.473931558 +0000 UTC m=+0.126325945 container create 360ad32c66603908e450129d85ddb7f64edd547c4d608995c9148680c5217216 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:49.722 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 podman[75570]: 2024-04-26 04:43:49.69327008 +0000 UTC m=+0.345664459 container init 360ad32c66603908e450129d85ddb7f64edd547c4d608995c9148680c5217216 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:49.722 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 podman[75570]: 2024-04-26 04:43:49.698890966 +0000 UTC m=+0.351285345 container start 360ad32c66603908e450129d85ddb7f64edd547c4d608995c9148680c5217216 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:43:49.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-mon[31139]: pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:50.001 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.722Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T04:43:50.001 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.723Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T04:43:50.001 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.723Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi171 (none))" 2024-04-26T04:43:50.001 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.723Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T04:43:50.001 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.723Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T04:43:50.002 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.724Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T04:43:50.002 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.724Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T04:43:50.002 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.725Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T04:43:50.002 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.725Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T04:43:50.002 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.728Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T04:43:50.002 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.728Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.032µs 2024-04-26T04:43:50.003 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.728Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T04:43:50.003 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.728Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-26T04:43:50.003 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.743Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-26T04:43:50.003 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.744Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-26T04:43:50.003 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.744Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=23.611µs wal_replay_duration=15.781915ms wbl_replay_duration=110ns total_replay_duration=15.824462ms 2024-04-26T04:43:50.003 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.746Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T04:43:50.004 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.746Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T04:43:50.004 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.746Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T04:43:50.004 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.765Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.757808ms db_storage=801ns remote_storage=1.182µs web_handler=351ns query_engine=701ns scrape=394.205µs scrape_sd=78.464µs notify=10.268µs notify_sd=8.46µs rules=17.951104ms tracing=6.29µs 2024-04-26T04:43:50.005 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.765Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T04:43:50.005 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:43:49.765Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T04:43:50.005 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 bash[75570]: 360ad32c66603908e450129d85ddb7f64edd547c4d608995c9148680c5217216 2024-04-26T04:43:50.005 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:43:49 smithi171 systemd[1]: Started Ceph prometheus.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:43:50.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:49 smithi156 ceph-mon[27570]: pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:50.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:49 smithi156 ceph-mon[39130]: pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:51.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:50 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:51.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:50 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:51.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:50 smithi156 ceph-mon[27570]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T04:43:51.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:50 smithi156 ceph-mon[27570]: Reconfiguring daemon grafana.a on smithi171 2024-04-26T04:43:51.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:50 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2269896315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:51.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:50 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:51.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:50 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:51.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:50 smithi156 ceph-mon[39130]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T04:43:51.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:50 smithi156 ceph-mon[39130]: Reconfiguring daemon grafana.a on smithi171 2024-04-26T04:43:51.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:50 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2269896315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:51.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:50 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:51.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:50 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:43:51.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:50 smithi171 ceph-mon[31139]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T04:43:51.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:50 smithi171 ceph-mon[31139]: Reconfiguring daemon grafana.a on smithi171 2024-04-26T04:43:51.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:50 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2269896315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:52.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:43:52 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:43:52.196Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002829048s 2024-04-26T04:43:52.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:52 smithi171 ceph-mon[31139]: pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:52.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:52 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3553669250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:53.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:52 smithi156 ceph-mon[27570]: pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:53.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:52 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3553669250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:53.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:52 smithi156 ceph-mon[39130]: pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:53.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:52 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3553669250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:54.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:54 smithi171 ceph-mon[31139]: pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:55.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:54 smithi156 ceph-mon[39130]: pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:55.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:54 smithi156 ceph-mon[27570]: pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:55.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:55 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3234444983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:56.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:55 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3234444983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:56.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:55 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3234444983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:56.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:56 smithi171 ceph-mon[31139]: pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:57.027 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:56 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:57.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:56 smithi156 ceph-mon[27570]: pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:57.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:56 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:57.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:56 smithi156 ceph-mon[39130]: pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:57.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:56 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:57.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:57 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3239841112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:58.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:57 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3239841112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:58.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:57 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3239841112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:58.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:58 smithi171 ceph-mon[31139]: pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:59.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:58 smithi156 ceph-mon[27570]: pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:59.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:58 smithi156 ceph-mon[39130]: pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:59.679 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:43:59 smithi171 systemd[1]: Stopping Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:43:59.992 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:43:59 smithi171 ceph-mon[31139]: pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:43:59.993 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:43:59 smithi171 podman[75977]: 2024-04-26 04:43:59.696254451 +0000 UTC m=+0.139242540 container died d42cfd971c1c00d9d339c7dfa97b5902451a2466958c5567cf5abecd0c1112a0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-26T04:44:00.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:43:59 smithi156 ceph-mon[27570]: pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:00.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:43:59 smithi156 ceph-mon[39130]: pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:00.249 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:00 smithi171 podman[75977]: 2024-04-26 04:44:00.073914074 +0000 UTC m=+0.516902160 container cleanup d42cfd971c1c00d9d339c7dfa97b5902451a2466958c5567cf5abecd0c1112a0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, vcs-type=git, com.redhat.component=ubi8-container, release=239.1651231664, version=8.5, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, io.openshift.expose-services=, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-26T04:44:00.250 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:00 smithi171 bash[75977]: ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a 2024-04-26T04:44:00.678 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:00 smithi171 podman[75996]: 2024-04-26 04:44:00.266997906 +0000 UTC m=+0.574016228 container remove d42cfd971c1c00d9d339c7dfa97b5902451a2466958c5567cf5abecd0c1112a0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, maintainer=Paul Cuzner , architecture=x86_64, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, vendor=Red Hat, Inc., description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, vcs-type=git, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:44:00.678 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:00 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:44:00.678 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:00 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:44:00.678 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:00 smithi171 systemd[1]: Stopped Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:00.678 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:00 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Consumed 5.460s CPU time. 2024-04-26T04:44:00.678 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:00 smithi171 systemd[1]: Starting Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:44:00.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:00 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3759566928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:01.000 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:00 smithi171 podman[76095]: 2024-04-26 04:44:00.600382631 +0000 UTC m=+0.017312592 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:44:01.000 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:00 smithi171 podman[76095]: 2024-04-26 04:44:00.701295195 +0000 UTC m=+0.118225193 container create 5d2586d050fcad55a14397fd934e46c0920319b8a613d3708dcde61bdb6f1f9e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, name=ubi8, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, vendor=Red Hat, Inc., io.buildah.version=1.32.0) 2024-04-26T04:44:01.000 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:00 smithi171 podman[76095]: 2024-04-26 04:44:00.920250785 +0000 UTC m=+0.337180737 container init 5d2586d050fcad55a14397fd934e46c0920319b8a613d3708dcde61bdb6f1f9e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, release=239.1651231664, vendor=Red Hat, Inc., vcs-type=git, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8) 2024-04-26T04:44:01.000 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:00 smithi171 podman[76095]: 2024-04-26 04:44:00.92606732 +0000 UTC m=+0.342997273 container start 5d2586d050fcad55a14397fd934e46c0920319b8a613d3708dcde61bdb6f1f9e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, version=8.5, vcs-type=git, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, io.openshift.expose-services=, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, maintainer=Paul Cuzner , name=ubi8, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-26T04:44:01.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:00 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3759566928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:01.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:00 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3759566928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:01.432 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:01 smithi171 bash[76095]: 5d2586d050fcad55a14397fd934e46c0920319b8a613d3708dcde61bdb6f1f9e 2024-04-26T04:44:01.432 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:01 smithi171 systemd[1]: Started Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:01.433 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[76110]: logger=settings t=2024-04-26T04:44:01.094732894Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:44:01.433 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:01 smithi171 podman[76146]: 2024-04-26 04:44:01.12106264 +0000 UTC m=+0.019086302 container died 5d2586d050fcad55a14397fd934e46c0920319b8a613d3708dcde61bdb6f1f9e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, release=239.1651231664, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, architecture=x86_64, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., description=Ceph Grafana Container, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner ) 2024-04-26T04:44:01.707 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:44:01] "GET /metrics HTTP/1.1" 200 33717 "" "Prometheus/2.43.0" 2024-04-26T04:44:02.002 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:01 smithi171 systemd[1]: Stopping Ceph node-exporter.b for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:44:02.002 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:01 smithi171 podman[76146]: 2024-04-26 04:44:01.706772275 +0000 UTC m=+0.604795924 container remove 5d2586d050fcad55a14397fd934e46c0920319b8a613d3708dcde61bdb6f1f9e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.openshift.tags=base rhel8, architecture=x86_64, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , io.buildah.version=1.32.0, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container) 2024-04-26T04:44:02.002 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:01 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:44:02.002 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:01 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:44:02.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:02 smithi156 ceph-mon[27570]: pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:02.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:02 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:02.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:02 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:02.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:02 smithi156 ceph-mon[27570]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T04:44:02.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:02 smithi156 ceph-mon[27570]: Reconfiguring daemon node-exporter.b on smithi171 2024-04-26T04:44:02.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:02 smithi156 ceph-mon[39130]: pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:02.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:02 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:02.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:02 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:02.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:02 smithi156 ceph-mon[39130]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T04:44:02.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:02 smithi156 ceph-mon[39130]: Reconfiguring daemon node-exporter.b on smithi171 2024-04-26T04:44:02.378 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:02 smithi171 podman[76312]: 2024-04-26 04:44:02.091052607 +0000 UTC m=+0.152129383 container died 5ef699e7d857d217f9bf8a73a55e118a6613daddd1f25d073a93edcb222c1657 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:44:02.379 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:02 smithi171 ceph-mon[31139]: pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:02.379 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:02 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:02.379 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:02 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:02.379 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:02 smithi171 ceph-mon[31139]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T04:44:02.379 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:02 smithi171 ceph-mon[31139]: Reconfiguring daemon node-exporter.b on smithi171 2024-04-26T04:44:02.694 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:02 smithi171 podman[76312]: 2024-04-26 04:44:02.495219753 +0000 UTC m=+0.556296530 container cleanup 5ef699e7d857d217f9bf8a73a55e118a6613daddd1f25d073a93edcb222c1657 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:44:02.694 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:02 smithi171 bash[76312]: ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b 2024-04-26T04:44:02.999 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:02 smithi171 podman[76332]: 2024-04-26 04:44:02.694501795 +0000 UTC m=+0.608058091 container remove 5ef699e7d857d217f9bf8a73a55e118a6613daddd1f25d073a93edcb222c1657 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:44:02.999 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:02 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:44:02.999 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:02 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-26T04:44:03.000 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:02 smithi171 systemd[1]: Stopped Ceph node-exporter.b for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:03.000 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:02 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 2.189s CPU time. 2024-04-26T04:44:03.000 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:02 smithi171 systemd[1]: Starting Ceph node-exporter.b for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:44:03.322 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/514171103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:03.323 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 podman[76433]: 2024-04-26 04:44:02.958010581 +0000 UTC m=+0.018218684 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:44:03.323 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 podman[76433]: 2024-04-26 04:44:03.078776208 +0000 UTC m=+0.138984297 container create a2d37cfca8cf7132ee7252fe19648f08ec8ff7fe47ebf6a819fc178a7f8340e7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:44:03.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:03 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/514171103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:03.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:03 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/514171103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:03.573 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE Bus STOPPING 2024-04-26T04:44:03.574 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:44:03.574 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE Bus STOPPED 2024-04-26T04:44:03.574 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE Bus STARTING 2024-04-26T04:44:03.574 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE Serving on http://:::9283 2024-04-26T04:44:03.574 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE Bus STARTED 2024-04-26T04:44:03.574 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 podman[76433]: 2024-04-26 04:44:03.322318857 +0000 UTC m=+0.382526945 container init a2d37cfca8cf7132ee7252fe19648f08ec8ff7fe47ebf6a819fc178a7f8340e7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:44:03.575 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 podman[76433]: 2024-04-26 04:44:03.327404591 +0000 UTC m=+0.387612682 container start a2d37cfca8cf7132ee7252fe19648f08ec8ff7fe47ebf6a819fc178a7f8340e7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:44:03.575 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.330Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:44:03.575 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.330Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:44:03.575 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.331Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:44:03.575 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.331Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:44:03.575 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.331Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:44:03.575 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.331Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:44:03.575 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:44:03.575 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:44:03.575 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:44:03.575 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:44:03.575 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:44:03.576 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:44:03.576 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:44:03.576 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:44:03.576 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:44:03.576 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:44:03.576 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:44:03.576 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:44:03.576 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:44:03.576 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:44:03.576 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:44:03.576 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:44:03.576 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:44:03.576 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:44:03.576 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:44:03.576 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:44:03.577 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:44:03.577 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:44:03.577 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:44:03.577 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:44:03.577 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:44:03.577 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:44:03.577 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:44:03.577 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:44:03.577 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:44:03.577 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:44:03.577 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:44:03.577 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:44:03.577 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:44:03.577 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:44:03.578 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:44:03.578 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:44:03.578 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:44:03.578 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:44:03.578 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:44:03.578 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:44:03.578 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:44:03.578 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:44:03.578 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:44:03.578 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:44:03.578 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:44:03.578 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:44:03.578 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b[76448]: ts=2024-04-26T04:44:03.332Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:44:03.578 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 bash[76433]: a2d37cfca8cf7132ee7252fe19648f08ec8ff7fe47ebf6a819fc178a7f8340e7 2024-04-26T04:44:03.579 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:44:03 smithi171 systemd[1]: Started Ceph node-exporter.b for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:03.999 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE Bus STOPPING 2024-04-26T04:44:04.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:44:04.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE Bus STOPPED 2024-04-26T04:44:04.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE Bus STARTING 2024-04-26T04:44:04.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE Serving on http://:::9283 2024-04-26T04:44:04.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE Bus STARTED 2024-04-26T04:44:04.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE Bus STOPPING 2024-04-26T04:44:04.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:44:04.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE Bus STOPPED 2024-04-26T04:44:04.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE Bus STARTING 2024-04-26T04:44:04.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE Serving on http://:::9283 2024-04-26T04:44:04.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:03] ENGINE Bus STARTED 2024-04-26T04:44:04.738 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:04.738 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:04.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:04.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:44:04.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:44:04.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:44:04.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:44:04.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:04.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:04.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:04.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171:9095"}]: dispatch 2024-04-26T04:44:04.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171:9095"}]: dispatch 2024-04-26T04:44:04.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:04.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:44:04.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:44:04.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171:3000"}]: dispatch 2024-04-26T04:44:04.740 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171:3000"}]: dispatch 2024-04-26T04:44:04.740 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:04.740 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:04.740 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:44:04.740 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:04 smithi171 ceph-mon[31139]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:44:04.797 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:04.797 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:04.797 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:04.797 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:44:04.797 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:44:04.798 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:44:04.798 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:44:04.798 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:04.798 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:04.798 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:04.798 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171:9095"}]: dispatch 2024-04-26T04:44:04.798 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171:9095"}]: dispatch 2024-04-26T04:44:04.798 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:04.798 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:44:04.798 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:44:04.798 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171:3000"}]: dispatch 2024-04-26T04:44:04.798 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171:3000"}]: dispatch 2024-04-26T04:44:04.799 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:04.799 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:04.799 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:44:04.799 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[27570]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:44:04.799 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:04.799 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:04.799 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:04.800 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:44:04.800 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:44:04.800 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:44:04.800 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:44:04.800 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:04.800 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:04.800 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:04.800 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171:9095"}]: dispatch 2024-04-26T04:44:04.800 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171:9095"}]: dispatch 2024-04-26T04:44:04.800 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:04.800 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:44:04.801 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:44:04.801 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171:3000"}]: dispatch 2024-04-26T04:44:04.801 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171:3000"}]: dispatch 2024-04-26T04:44:04.801 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:04.801 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:04.801 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:44:04.801 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:04 smithi156 ceph-mon[39130]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:44:05.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:05 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1411272558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:05.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:05 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:05.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:05 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:05.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:05 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:05.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:05 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:44:05.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:05 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:05.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:05 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1411272558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:05.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:05 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:05.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:05 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:05.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:05 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:05.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:05 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:44:05.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:05 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:05.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:05 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1411272558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:05.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:05 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:05.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:05 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:05.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:05 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:05.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:05 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:44:05.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:05 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:06.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:06 smithi156 ceph-mon[27570]: pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:06.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:06 smithi156 ceph-mon[27570]: Upgrade: Updating mgr.y 2024-04-26T04:44:06.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:06 smithi156 ceph-mon[27570]: Deploying daemon mgr.y on smithi156 2024-04-26T04:44:06.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:06 smithi156 ceph-mon[39130]: pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:06.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:06 smithi156 ceph-mon[39130]: Upgrade: Updating mgr.y 2024-04-26T04:44:06.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:06 smithi156 ceph-mon[39130]: Deploying daemon mgr.y on smithi156 2024-04-26T04:44:06.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:06 smithi171 ceph-mon[31139]: pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:06.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:06 smithi171 ceph-mon[31139]: Upgrade: Updating mgr.y 2024-04-26T04:44:06.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:06 smithi171 ceph-mon[31139]: Deploying daemon mgr.y on smithi156 2024-04-26T04:44:07.518 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:07 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/345047702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:07.519 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:07 smithi156 systemd[1]: Stopping Ceph mgr.y for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:44:07.519 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:07 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/345047702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:07.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:07 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/345047702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:07.776 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:07 smithi156 podman[89656]: 2024-04-26 04:44:07.517358547 +0000 UTC m=+0.137979330 container died b232ee6793f9c41d372c6cb52800846e66d9cf46871d614441f698cf5b8e2c4a (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-26T04:44:08.060 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:07 smithi156 podman[89656]: 2024-04-26 04:44:07.849797066 +0000 UTC m=+0.470417840 container cleanup b232ee6793f9c41d372c6cb52800846e66d9cf46871d614441f698cf5b8e2c4a (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-26T04:44:08.060 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:07 smithi156 bash[89656]: ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y 2024-04-26T04:44:08.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:08 smithi156 podman[89673]: 2024-04-26 04:44:08.059793422 +0000 UTC m=+0.542633010 container remove b232ee6793f9c41d372c6cb52800846e66d9cf46871d614441f698cf5b8e2c4a (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-26T04:44:08.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:08 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:44:08.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:08 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-04-26T04:44:08.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:08 smithi156 systemd[1]: Stopped Ceph mgr.y for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:08.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:08 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.y.service: Consumed 58.019s CPU time. 2024-04-26T04:44:08.683 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:08 smithi156 systemd[1]: Starting Ceph mgr.y for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:44:08.683 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:08 smithi156 ceph-mon[27570]: pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:08.683 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:08 smithi156 ceph-mon[39130]: pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:08.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:08 smithi171 ceph-mon[31139]: pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:08.750 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:08 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:44:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:44:08.935 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:08 smithi156 podman[89807]: 2024-04-26 04:44:08.586103782 +0000 UTC m=+0.018317673 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:44:08.935 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:08 smithi156 podman[89807]: 2024-04-26 04:44:08.685800601 +0000 UTC m=+0.118014483 container create 715bee4dd354b9b2a0844de7cd979c1044b2c29962121334105eb8f66f5a8c2b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:44:08.935 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:08 smithi156 podman[89807]: 2024-04-26 04:44:08.929909758 +0000 UTC m=+0.362123644 container init 715bee4dd354b9b2a0844de7cd979c1044b2c29962121334105eb8f66f5a8c2b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5) 2024-04-26T04:44:08.936 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:08 smithi156 podman[89807]: 2024-04-26 04:44:08.935154564 +0000 UTC m=+0.367368447 container start 715bee4dd354b9b2a0844de7cd979c1044b2c29962121334105eb8f66f5a8c2b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-26T04:44:09.299 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:08 smithi156 ceph-mgr[89844]: -- 172.21.15.156:0/907334982 <== mon.0 v2:172.21.15.156:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5605e6b005a0 con 0x5605e6ad4800 2024-04-26T04:44:09.299 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:09 smithi156 bash[89807]: 715bee4dd354b9b2a0844de7cd979c1044b2c29962121334105eb8f66f5a8c2b 2024-04-26T04:44:09.299 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:09 smithi156 systemd[1]: Started Ceph mgr.y for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:09.299 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:09 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:09.091+0000 7fe21dd34200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:44:09.299 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:09 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:09.296+0000 7fe21dd34200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:44:09.593 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:09 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:09.413+0000 7fe21dd34200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:44:10.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:09 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:09.765+0000 7fe21dd34200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:44:10.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:09 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:09.860+0000 7fe21dd34200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:44:10.335 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:10 smithi171 ceph-mon[31139]: pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:10.335 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:10 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:10.335 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:10 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:10.335 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:10 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:10.335 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:10 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2484416616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:10.451 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:10 smithi156 ceph-mon[27570]: pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:10.451 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:10 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:10.451 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:10 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:10.451 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:10 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:10.452 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:10 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2484416616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:10.452 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:10 smithi156 ceph-mon[39130]: pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:10.452 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:10 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:10.452 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:10 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:10.452 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:10 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:10.452 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:10 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2484416616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:11.321 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:11 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:11.321 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:11 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:11.142+0000 7fe21dd34200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:44:11.321 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:11 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:11.319+0000 7fe21dd34200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:44:11.321 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:11 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:11.434 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:11 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:11.593 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:11 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:11.513+0000 7fe21dd34200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:44:11.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:11 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:44:11] "GET /metrics HTTP/1.1" 200 33717 "" "Prometheus/2.43.0" 2024-04-26T04:44:12.158 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:11 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:11.845+0000 7fe21dd34200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:44:12.158 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:11 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:11.938+0000 7fe21dd34200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:44:12.173 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:12 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 1. 2024-04-26T04:44:12.173 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:12 smithi171 systemd[1]: Stopped Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:12.173 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:12 smithi171 systemd[1]: Starting Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:44:12.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:12 smithi171 ceph-mon[31139]: pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:12.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:12 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3735017306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:12.500 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:12 smithi171 podman[77003]: 2024-04-26 04:44:12.293144393 +0000 UTC m=+0.021824135 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:44:12.500 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:12 smithi171 podman[77003]: 2024-04-26 04:44:12.401340405 +0000 UTC m=+0.130020188 container create 1a942f82440262063f1451d08d3d1728d0caf2705080b5f759bc0bfc79eb6238 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.component=ubi8-container, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, io.buildah.version=1.32.0, name=ubi8, release=239.1651231664, io.openshift.tags=base rhel8) 2024-04-26T04:44:12.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:12 smithi156 ceph-mon[27570]: pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:12.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:12 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3735017306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:12.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:12 smithi156 ceph-mon[39130]: pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:12.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:12 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3735017306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:12.594 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:12 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:12.157+0000 7fe21dd34200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:44:12.594 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:12 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:12.359+0000 7fe21dd34200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:44:12.999 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:12 smithi171 podman[77003]: 2024-04-26 04:44:12.594829452 +0000 UTC m=+0.323509195 container init 1a942f82440262063f1451d08d3d1728d0caf2705080b5f759bc0bfc79eb6238 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, version=8.5, release=239.1651231664, architecture=x86_64, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, vcs-type=git, name=ubi8, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-26T04:44:12.999 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:12 smithi171 podman[77003]: 2024-04-26 04:44:12.600814613 +0000 UTC m=+0.329494356 container start 1a942f82440262063f1451d08d3d1728d0caf2705080b5f759bc0bfc79eb6238 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, description=Ceph Grafana Container, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , version=8.5, build-date=2022-04-29T11:48:11.055897, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, distribution-scope=public) 2024-04-26T04:44:13.000 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:12 smithi171 bash[77003]: 1a942f82440262063f1451d08d3d1728d0caf2705080b5f759bc0bfc79eb6238 2024-04-26T04:44:13.000 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:12 smithi171 systemd[1]: Started Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:13.000 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:12 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[77019]: logger=settings t=2024-04-26T04:44:12.782409042Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:44:13.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:12 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:12.985+0000 7fe21dd34200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:44:13.368 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:13 smithi171 podman[77103]: 2024-04-26 04:44:13.046982827 +0000 UTC m=+0.254943626 container died 1a942f82440262063f1451d08d3d1728d0caf2705080b5f759bc0bfc79eb6238 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, io.openshift.tags=base rhel8, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.buildah.version=1.32.0, architecture=x86_64, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., release=239.1651231664, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-26T04:44:13.761 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:13 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:13.436+0000 7fe21dd34200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:44:13.761 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:13 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:13.541+0000 7fe21dd34200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:44:13.761 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:13 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:13.760+0000 7fe21dd34200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:44:14.249 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:13 smithi171 podman[77103]: 2024-04-26 04:44:13.761718223 +0000 UTC m=+0.969679019 container remove 1a942f82440262063f1451d08d3d1728d0caf2705080b5f759bc0bfc79eb6238 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, maintainer=Paul Cuzner , architecture=x86_64, com.redhat.component=ubi8-container, io.openshift.expose-services=, vendor=Red Hat, Inc., version=8.5, release=239.1651231664, vcs-type=git, io.buildah.version=1.32.0, name=ubi8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-26T04:44:14.249 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:13 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:44:14.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:14 smithi156 ceph-mon[39130]: pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:14.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:14 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:14.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:14 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:14.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:14 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1948823922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:14.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:14 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:14.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:14 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:14.594 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:14.320+0000 7fe21dd34200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:44:14.594 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:14.422+0000 7fe21dd34200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:44:14.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:14 smithi156 ceph-mon[27570]: pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:14.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:14 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:14.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:14 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:14.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:14 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1948823922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:14.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:14 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:14.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:14 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:14.749 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:14 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:44:14.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:14 smithi171 ceph-mon[31139]: pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:14.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:14 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:14.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:14 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:14.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:14 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1948823922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:14.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:14 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:14.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:14 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:14.940 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:14.686+0000 7fe21dd34200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:44:14.940 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:14.830+0000 7fe21dd34200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:44:14.940 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:14.938+0000 7fe21dd34200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:44:15.244 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:15.067+0000 7fe21dd34200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:44:15.244 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:15.163+0000 7fe21dd34200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:44:15.510 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:15 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:15.511 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:15 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:15.511 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:15 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:15.511 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:15 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:15.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:15 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:15.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:15 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:16.234 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:16.010+0000 7fe21dd34200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:44:16.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-mon[39130]: pgmap v35: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:16.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-mon[39130]: Standby manager daemon y restarted 2024-04-26T04:44:16.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-mon[39130]: Standby manager daemon y started 2024-04-26T04:44:16.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:44:16.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:44:16.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:44:16.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:44:16.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-mon[27570]: pgmap v35: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:16.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-mon[27570]: Standby manager daemon y restarted 2024-04-26T04:44:16.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-mon[27570]: Standby manager daemon y started 2024-04-26T04:44:16.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:44:16.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:44:16.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:44:16.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:44:16.595 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:16.233+0000 7fe21dd34200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:44:16.595 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:44:16] ENGINE Bus STARTING 2024-04-26T04:44:16.596 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: CherryPy Checker: 2024-04-26T04:44:16.596 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: The Application mounted at '' has an empty config. 2024-04-26T04:44:16.596 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:16.596 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:44:16] ENGINE Serving on http://:::9283 2024-04-26T04:44:16.596 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:16 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:44:16] ENGINE Bus STARTED 2024-04-26T04:44:16.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:16 smithi171 ceph-mon[31139]: pgmap v35: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:16.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:16 smithi171 ceph-mon[31139]: Standby manager daemon y restarted 2024-04-26T04:44:16.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:16 smithi171 ceph-mon[31139]: Standby manager daemon y started 2024-04-26T04:44:16.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:16 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:44:16.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:16 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:44:16.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:16 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:44:16.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:16 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:44:17.747 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:17 smithi156 ceph-mon[39130]: mgrmap e27: x(active, since 65s), standbys: y 2024-04-26T04:44:17.747 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:17 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1259250367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:17.748 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:17 smithi156 ceph-mon[27570]: mgrmap e27: x(active, since 65s), standbys: y 2024-04-26T04:44:17.748 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:17 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1259250367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:17.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:17 smithi171 ceph-mon[31139]: mgrmap e27: x(active, since 65s), standbys: y 2024-04-26T04:44:17.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:17 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1259250367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:18.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:44:18] ENGINE Bus STOPPING 2024-04-26T04:44:18.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:44:18] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:44:18.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:44:18] ENGINE Bus STOPPED 2024-04-26T04:44:18.660 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:18.313+0000 7ff228496700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-26T04:44:18.660 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: ignoring --setuser ceph since I am not root 2024-04-26T04:44:18.660 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: ignoring --setgroup ceph since I am not root 2024-04-26T04:44:18.660 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mgr[66767]: -- 172.21.15.171:0/1547808168 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559ac7c745a0 con 0x559ac7c48c00 2024-04-26T04:44:18.660 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:18.494+0000 7f952e656200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:44:18.661 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: pgmap v36: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:18.661 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:18.661 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:18.661 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:18.661 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:44:18.661 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:18.661 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:18.661 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:44:18.661 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:44:18.661 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: Activating manager daemon y 2024-04-26T04:44:18.662 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:44:18.662 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.24691 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T04:44:18.662 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: mgrmap e28: y(active, starting, since 0.00600657s) 2024-04-26T04:44:18.662 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:44:18.662 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:44:18.662 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:44:18.662 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.wyfzii"}]: dispatch 2024-04-26T04:44:18.662 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.kzjlqn"}]: dispatch 2024-04-26T04:44:18.662 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:44:18.662 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:44:18.662 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:44:18.663 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:44:18.663 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:44:18.663 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:44:18.663 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:44:18.663 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:44:18.663 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:44:18.663 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:44:18.663 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:44:18.663 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:44:18.663 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: Manager daemon y is now available 2024-04-26T04:44:18.663 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:18.663 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:18.664 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:44:18.664 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:44:18.664 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:44:18.664 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:44:18.844 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:44:18] ENGINE Bus STARTING 2024-04-26T04:44:18.844 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:18.446+0000 7fe17e925700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:44:18.844 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:18.446+0000 7fe17e925700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:44:18.844 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:18.446+0000 7fe17e925700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:44:18.844 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:18.446+0000 7fe17e925700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:44:18.844 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:18.446+0000 7fe17e925700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:44:18.844 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:18.446+0000 7fe17b11e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:44:18.844 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:18.446+0000 7fe17b11e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:44:18.844 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:18.446+0000 7fe17b11e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:44:18.844 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:18.446+0000 7fe17b11e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:44:18.844 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:18.446+0000 7fe17b11e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:44:18.844 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:44:18] ENGINE Serving on http://:::9283 2024-04-26T04:44:18.845 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:44:18] ENGINE Bus STARTED 2024-04-26T04:44:18.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: pgmap v36: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:18.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:18.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:18.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:18.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:44:18.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:18.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:18.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:44:18.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:44:18.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: Activating manager daemon y 2024-04-26T04:44:18.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:44:18.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.24691 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T04:44:18.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: mgrmap e28: y(active, starting, since 0.00600657s) 2024-04-26T04:44:18.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:44:18.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:44:18.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:44:18.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.wyfzii"}]: dispatch 2024-04-26T04:44:18.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.kzjlqn"}]: dispatch 2024-04-26T04:44:18.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:44:18.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:44:18.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:44:18.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:44:18.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:44:18.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:44:18.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:44:18.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:44:18.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:44:18.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:44:18.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:44:18.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:44:18.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: Manager daemon y is now available 2024-04-26T04:44:18.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:18.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:18.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:44:18.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:44:18.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:44:18.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:44:18.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: pgmap v36: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:18.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:18.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:18.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:18.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:44:18.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' 2024-04-26T04:44:18.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:18.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.24691 172.21.15.171:0/918841629' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:44:18.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:44:18.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: Activating manager daemon y 2024-04-26T04:44:18.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:44:18.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.24691 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T04:44:18.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: mgrmap e28: y(active, starting, since 0.00600657s) 2024-04-26T04:44:18.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:44:18.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:44:18.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:44:18.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.wyfzii"}]: dispatch 2024-04-26T04:44:18.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.kzjlqn"}]: dispatch 2024-04-26T04:44:18.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:44:18.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:44:18.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:44:18.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:44:18.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:44:18.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:44:18.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:44:18.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:44:18.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:44:18.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:44:18.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:44:18.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:44:18.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: Manager daemon y is now available 2024-04-26T04:44:18.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:18.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:18.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:44:18.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:44:18.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:44:18.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:18 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:44:18.999 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:18.683+0000 7f952e656200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:44:18.999 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:18.790+0000 7f952e656200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:44:19.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:18 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:44:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:44:19.486 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:19.113+0000 7f952e656200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:44:19.487 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:19.218+0000 7f952e656200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:44:19.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:19 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/4266102542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:19.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:19 smithi171 ceph-mon[31139]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:44:19.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:19 smithi171 ceph-mon[31139]: mgrmap e29: y(active, since 1.00766s) 2024-04-26T04:44:19.815 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:19 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/4266102542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:19.815 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:19 smithi156 ceph-mon[27570]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:44:19.816 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:19 smithi156 ceph-mon[27570]: mgrmap e29: y(active, since 1.00766s) 2024-04-26T04:44:19.816 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:19 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/4266102542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:19.816 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:19 smithi156 ceph-mon[39130]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:44:19.816 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:19 smithi156 ceph-mon[39130]: mgrmap e29: y(active, since 1.00766s) 2024-04-26T04:44:20.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:20 smithi156 ceph-mon[39130]: [26/Apr/2024:04:44:19] ENGINE Bus STARTING 2024-04-26T04:44:20.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:20 smithi156 ceph-mon[39130]: [26/Apr/2024:04:44:19] ENGINE Serving on https://172.21.15.156:7150 2024-04-26T04:44:20.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:20 smithi156 ceph-mon[39130]: [26/Apr/2024:04:44:19] ENGINE Serving on http://172.21.15.156:8765 2024-04-26T04:44:20.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:20 smithi156 ceph-mon[39130]: [26/Apr/2024:04:44:19] ENGINE Bus STARTED 2024-04-26T04:44:20.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:20 smithi156 ceph-mon[39130]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:44:20.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:20 smithi156 ceph-mon[39130]: Cluster is now healthy 2024-04-26T04:44:20.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:20 smithi156 ceph-mon[27570]: [26/Apr/2024:04:44:19] ENGINE Bus STARTING 2024-04-26T04:44:20.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:20 smithi156 ceph-mon[27570]: [26/Apr/2024:04:44:19] ENGINE Serving on https://172.21.15.156:7150 2024-04-26T04:44:20.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:20 smithi156 ceph-mon[27570]: [26/Apr/2024:04:44:19] ENGINE Serving on http://172.21.15.156:8765 2024-04-26T04:44:20.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:20 smithi156 ceph-mon[27570]: [26/Apr/2024:04:44:19] ENGINE Bus STARTED 2024-04-26T04:44:20.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:20 smithi156 ceph-mon[27570]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:44:20.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:20 smithi156 ceph-mon[27570]: Cluster is now healthy 2024-04-26T04:44:20.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:20 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:20.456+0000 7f952e656200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:44:20.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:20 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:20.653+0000 7f952e656200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:44:20.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:20 smithi171 ceph-mon[31139]: [26/Apr/2024:04:44:19] ENGINE Bus STARTING 2024-04-26T04:44:20.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:20 smithi171 ceph-mon[31139]: [26/Apr/2024:04:44:19] ENGINE Serving on https://172.21.15.156:7150 2024-04-26T04:44:20.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:20 smithi171 ceph-mon[31139]: [26/Apr/2024:04:44:19] ENGINE Serving on http://172.21.15.156:8765 2024-04-26T04:44:20.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:20 smithi171 ceph-mon[31139]: [26/Apr/2024:04:44:19] ENGINE Bus STARTED 2024-04-26T04:44:20.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:20 smithi171 ceph-mon[31139]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:44:20.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:20 smithi171 ceph-mon[31139]: Cluster is now healthy 2024-04-26T04:44:21.134 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:20 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:20.828+0000 7f952e656200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:44:21.135 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:21.133+0000 7f952e656200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:44:21.452 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:21.239+0000 7f952e656200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:44:21.452 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:21.451+0000 7f952e656200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:44:21.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:21 smithi156 ceph-mon[39130]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:21.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:21 smithi156 ceph-mon[39130]: mgrmap e30: y(active, since 2s) 2024-04-26T04:44:21.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:21 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2638795478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:21.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:21 smithi156 ceph-mon[27570]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:21.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:21 smithi156 ceph-mon[27570]: mgrmap e30: y(active, since 2s) 2024-04-26T04:44:21.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:21 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2638795478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:21.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:21 smithi171 ceph-mon[31139]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:21.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:21 smithi171 ceph-mon[31139]: mgrmap e30: y(active, since 2s) 2024-04-26T04:44:21.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:21 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2638795478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:21.750 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:21.658+0000 7f952e656200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:44:22.532 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:22 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:22.325+0000 7f952e656200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:44:23.118 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:22 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:22.789+0000 7f952e656200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:44:23.118 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:22 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:22.891+0000 7f952e656200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:44:23.118 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:23 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:23.117+0000 7f952e656200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:44:23.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:23 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:23.673+0000 7f952e656200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:44:23.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:23 smithi171 ceph-mon[31139]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:23.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:23 smithi171 ceph-mon[31139]: mgrmap e31: y(active, since 4s) 2024-04-26T04:44:23.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:23 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:23.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:23 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:23.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:23 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:23.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:23 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:23.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:23 smithi156 ceph-mon[27570]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:23.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:23 smithi156 ceph-mon[27570]: mgrmap e31: y(active, since 4s) 2024-04-26T04:44:23.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:23 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:23.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:23 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:23.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:23 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:23.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:23 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:23.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:23 smithi156 ceph-mon[39130]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:23.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:23 smithi156 ceph-mon[39130]: mgrmap e31: y(active, since 4s) 2024-04-26T04:44:23.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:23 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:23.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:23 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:23.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:23 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:23.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:23 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:24.034 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:23 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:23.773+0000 7f952e656200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:44:24.401 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:24 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:24.033+0000 7f952e656200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:44:24.401 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:24 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:24.176+0000 7f952e656200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:44:24.401 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:24 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:24.279+0000 7f952e656200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:44:24.401 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:24 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:24.400+0000 7f952e656200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:44:24.736 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:24 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:24.495+0000 7f952e656200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:44:24.736 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:24 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 2. 2024-04-26T04:44:24.736 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:24 smithi171 systemd[1]: Stopped Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:24.737 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:24 smithi171 systemd[1]: Starting Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:44:24.737 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:24 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:24.737 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:24 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:24.737 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:24 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2848886277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:24.737 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:24 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:24.737 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:24 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:24.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:24 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:24.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:24 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:24.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:24 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2848886277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:24.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:24 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:24.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:24 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:24.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:24 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:24.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:24 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:24.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:24 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2848886277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:24.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:24 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:24.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:24 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:25.000 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:24 smithi171 podman[78131]: 2024-04-26 04:44:24.638043993 +0000 UTC m=+0.019389894 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:44:25.000 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:24 smithi171 podman[78131]: 2024-04-26 04:44:24.778244771 +0000 UTC m=+0.159590663 container create ae0b2029f60a04ca4695182196948d92d009d5022af5865cd56464956834ce20 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, vcs-type=git, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, io.openshift.tags=base rhel8, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , version=8.5, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-26T04:44:25.380 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:25 smithi171 podman[78131]: 2024-04-26 04:44:25.055512684 +0000 UTC m=+0.436858576 container init ae0b2029f60a04ca4695182196948d92d009d5022af5865cd56464956834ce20 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, architecture=x86_64, vcs-type=git, io.buildah.version=1.32.0, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, version=8.5, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-26T04:44:25.380 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:25 smithi171 podman[78131]: 2024-04-26 04:44:25.061046587 +0000 UTC m=+0.442392480 container start ae0b2029f60a04ca4695182196948d92d009d5022af5865cd56464956834ce20 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , vcs-type=git, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, architecture=x86_64) 2024-04-26T04:44:25.380 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:25 smithi171 bash[78131]: ae0b2029f60a04ca4695182196948d92d009d5022af5865cd56464956834ce20 2024-04-26T04:44:25.380 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:25 smithi171 systemd[1]: Started Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:25.381 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[78147]: logger=settings t=2024-04-26T04:44:25.235881508Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:44:25.381 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:25 smithi171 podman[78185]: 2024-04-26 04:44:25.267778793 +0000 UTC m=+0.020177529 container died ae0b2029f60a04ca4695182196948d92d009d5022af5865cd56464956834ce20 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, vcs-type=git, vendor=Red Hat, Inc., version=8.5, io.openshift.tags=base rhel8, io.openshift.expose-services=, release=239.1651231664, architecture=x86_64, distribution-scope=public) 2024-04-26T04:44:25.631 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:25.379+0000 7f952e656200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:44:25.631 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:25.604+0000 7f952e656200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:44:25.631 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:25] ENGINE Bus STARTING 2024-04-26T04:44:25.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:25 smithi171 ceph-mon[31139]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:26.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:25 smithi171 ceph-mon[31139]: Standby manager daemon x started 2024-04-26T04:44:26.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:25 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/307342782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:44:26.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:25 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/307342782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:44:26.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:25 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/307342782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:44:26.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:25 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/307342782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:44:26.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: CherryPy Checker: 2024-04-26T04:44:26.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: The Application mounted at '' has an empty config. 2024-04-26T04:44:26.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: 2024-04-26T04:44:26.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:25] ENGINE Serving on http://:::9283 2024-04-26T04:44:26.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: [26/Apr/2024:04:44:25] ENGINE Bus STARTED 2024-04-26T04:44:26.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:25 smithi156 ceph-mon[27570]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:26.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:25 smithi156 ceph-mon[27570]: Standby manager daemon x started 2024-04-26T04:44:26.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:25 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/307342782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:44:26.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:25 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/307342782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:44:26.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:25 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/307342782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:44:26.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:25 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/307342782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:44:26.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:25 smithi156 ceph-mon[39130]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:26.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:25 smithi156 ceph-mon[39130]: Standby manager daemon x started 2024-04-26T04:44:26.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:25 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/307342782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:44:26.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:25 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/307342782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:44:26.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:25 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/307342782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:44:26.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:25 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/307342782' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:44:26.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:26 smithi156 ceph-mon[27570]: mgrmap e32: y(active, since 7s), standbys: x 2024-04-26T04:44:26.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:26 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:44:26.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:26 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3787324156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:26.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:26 smithi156 ceph-mon[39130]: mgrmap e32: y(active, since 7s), standbys: x 2024-04-26T04:44:26.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:26 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:44:26.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:26 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3787324156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:26.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:26 smithi171 ceph-mon[31139]: mgrmap e32: y(active, since 7s), standbys: x 2024-04-26T04:44:26.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:26 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:44:27.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:26 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3787324156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:27.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:27 smithi171 ceph-mon[31139]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:44:27.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:27 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:27.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:27 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:27.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:27 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:28.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:27 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:44:28.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:27 smithi156 ceph-mon[39130]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:44:28.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:27 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:28.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:27 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:28.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:27 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:28.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:27 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:44:28.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:27 smithi156 ceph-mon[27570]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:44:28.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:27 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:28.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:27 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:28.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:27 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:28.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:27 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:44:29.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:28 smithi171 ceph-mon[31139]: Updating smithi156:/etc/ceph/ceph.conf 2024-04-26T04:44:29.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:28 smithi171 ceph-mon[31139]: Updating smithi171:/etc/ceph/ceph.conf 2024-04-26T04:44:29.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:28 smithi171 ceph-mon[31139]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:29.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:28 smithi171 ceph-mon[31139]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:29.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:28 smithi171 ceph-mon[31139]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:29.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:28 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:29.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:28 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:29.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:28 smithi156 ceph-mon[27570]: Updating smithi156:/etc/ceph/ceph.conf 2024-04-26T04:44:29.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:28 smithi156 ceph-mon[27570]: Updating smithi171:/etc/ceph/ceph.conf 2024-04-26T04:44:29.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:28 smithi156 ceph-mon[27570]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:29.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:28 smithi156 ceph-mon[27570]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:29.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:28 smithi156 ceph-mon[27570]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:29.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:28 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:29.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:28 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:29.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:28 smithi156 ceph-mon[39130]: Updating smithi156:/etc/ceph/ceph.conf 2024-04-26T04:44:29.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:28 smithi156 ceph-mon[39130]: Updating smithi171:/etc/ceph/ceph.conf 2024-04-26T04:44:29.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:28 smithi156 ceph-mon[39130]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:29.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:28 smithi156 ceph-mon[39130]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:29.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:28 smithi156 ceph-mon[39130]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:29.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:28 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:29.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:28 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:29.499 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:29 smithi171 podman[78185]: 2024-04-26 04:44:29.174294475 +0000 UTC m=+3.926693213 container remove ae0b2029f60a04ca4695182196948d92d009d5022af5865cd56464956834ce20 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, release=239.1651231664, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.buildah.version=1.32.0, maintainer=Paul Cuzner , version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.openshift.tags=base rhel8, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, vcs-type=git) 2024-04-26T04:44:29.499 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:29 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:44:29.499 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:29 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:44:29.944 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:29 smithi171 ceph-mon[31139]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:29.944 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:29 smithi171 ceph-mon[31139]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:29.944 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:29 smithi171 ceph-mon[31139]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:44:29.944 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:29 smithi171 ceph-mon[31139]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:29.944 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:29 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:29.944 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:29 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:29.945 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:29 smithi171 ceph-mon[31139]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:44:29.945 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:29 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:29.945 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:29 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1509028155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:29.945 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:29 smithi171 ceph-mon[31139]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:44:29.945 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:29 smithi171 ceph-mon[31139]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:44:30.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[27570]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:30.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[27570]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:30.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[27570]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:44:30.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[27570]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:30.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:30.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:30.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[27570]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:44:30.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:30.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/1509028155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:30.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[27570]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:44:30.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[27570]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:44:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[39130]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[39130]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[39130]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:44:30.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[39130]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[39130]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:44:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1509028155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[39130]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:44:30.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:29 smithi156 ceph-mon[39130]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:44:30.884 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:30 smithi171 ceph-mon[31139]: Reconfiguring daemon prometheus.a on smithi171 2024-04-26T04:44:31.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:30 smithi156 ceph-mon[27570]: Reconfiguring daemon prometheus.a on smithi171 2024-04-26T04:44:31.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:30 smithi156 ceph-mon[39130]: Reconfiguring daemon prometheus.a on smithi171 2024-04-26T04:44:31.499 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:31 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[66744]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:44:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:44:31.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:31 smithi171 ceph-mon[31139]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:44:32.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:31 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3451750193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:32.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:31 smithi156 ceph-mon[27570]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:44:32.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:31 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3451750193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:32.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:31 smithi156 ceph-mon[39130]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:44:32.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:31 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3451750193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:32.706 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:32 smithi171 systemd[1]: Stopping Ceph prometheus.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:44:32.978 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:44:32.705Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:44:32.978 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:44:32.705Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T04:44:32.978 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:44:32.706Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T04:44:32.978 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:44:32.706Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T04:44:32.978 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:44:32.706Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T04:44:32.979 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:44:32.706Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T04:44:32.979 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:44:32.706Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T04:44:32.979 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:44:32.706Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T04:44:32.979 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:44:32.706Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T04:44:32.979 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:44:32.706Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T04:44:32.979 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:44:32.706Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T04:44:32.979 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[75584]: ts=2024-04-26T04:44:32.706Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T04:44:32.979 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:32 smithi171 podman[79367]: 2024-04-26 04:44:32.716450543 +0000 UTC m=+0.120893933 container died 360ad32c66603908e450129d85ddb7f64edd547c4d608995c9148680c5217216 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:33.249 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:33 smithi171 podman[79367]: 2024-04-26 04:44:33.111399647 +0000 UTC m=+0.515842993 container cleanup 360ad32c66603908e450129d85ddb7f64edd547c4d608995c9148680c5217216 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:33.250 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:33 smithi171 bash[79367]: ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T04:44:33.675 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:33 smithi171 podman[79386]: 2024-04-26 04:44:33.39580764 +0000 UTC m=+0.682068810 container remove 360ad32c66603908e450129d85ddb7f64edd547c4d608995c9148680c5217216 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:33.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:33 smithi171 ceph-mon[31139]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:44:34.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:33 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:34.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:33 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:34.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:33 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3268642208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:34.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:33 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T04:44:34.001 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:33 smithi171 systemd[1]: Stopped Ceph prometheus.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:34.001 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:33 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.194s CPU time. 2024-04-26T04:44:34.001 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:33 smithi171 systemd[1]: Starting Ceph prometheus.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:44:34.001 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:33 smithi171 podman[79486]: 2024-04-26 04:44:33.810407206 +0000 UTC m=+0.019156757 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T04:44:34.001 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:33 smithi171 podman[79486]: 2024-04-26 04:44:33.938872791 +0000 UTC m=+0.147622385 container create 93d6b22f6fcef0a3f36edba300841842ad87b33a57041fcd3f9d32679a67c9f7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:34.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:33 smithi156 ceph-mon[27570]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:44:34.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:33 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:34.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:33 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:34.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:33 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3268642208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:34.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:33 smithi156 ceph-mon[39130]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:44:34.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:33 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:34.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:33 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:34.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:33 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3268642208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:34.500 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 podman[79486]: 2024-04-26 04:44:34.165876919 +0000 UTC m=+0.374626471 container init 93d6b22f6fcef0a3f36edba300841842ad87b33a57041fcd3f9d32679a67c9f7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:34.500 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 podman[79486]: 2024-04-26 04:44:34.17129004 +0000 UTC m=+0.380039593 container start 93d6b22f6fcef0a3f36edba300841842ad87b33a57041fcd3f9d32679a67c9f7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:34.500 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.195Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T04:44:34.500 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.195Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T04:44:34.500 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.195Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi171 (none))" 2024-04-26T04:44:34.500 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.195Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T04:44:34.500 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.195Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T04:44:34.500 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.197Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T04:44:34.500 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.197Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T04:44:34.501 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.198Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T04:44:34.501 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.198Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T04:44:34.501 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.200Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T04:44:34.501 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.200Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.554µs 2024-04-26T04:44:34.501 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.200Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T04:44:34.501 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.201Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-04-26T04:44:34.501 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.215Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-04-26T04:44:34.501 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.217Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-04-26T04:44:34.501 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.217Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-04-26T04:44:34.501 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.218Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=30.518µs wal_replay_duration=17.336676ms wbl_replay_duration=423ns total_replay_duration=17.385345ms 2024-04-26T04:44:34.501 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.220Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T04:44:34.501 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.220Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T04:44:34.501 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.220Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T04:44:34.502 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.239Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.187419ms db_storage=834ns remote_storage=1.113µs web_handler=438ns query_engine=744ns scrape=352.581µs scrape_sd=99.502µs notify=18.457µs notify_sd=19.243µs rules=17.467142ms tracing=5.437µs 2024-04-26T04:44:34.502 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.239Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T04:44:34.502 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:34.239Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T04:44:34.502 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 bash[79486]: 93d6b22f6fcef0a3f36edba300841842ad87b33a57041fcd3f9d32679a67c9f7 2024-04-26T04:44:34.502 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:34 smithi171 systemd[1]: Started Ceph prometheus.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:34.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:44:34] ENGINE Bus STOPPING 2024-04-26T04:44:34.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:44:34] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:44:34.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:44:34] ENGINE Bus STOPPED 2024-04-26T04:44:34.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:44:34] ENGINE Bus STARTING 2024-04-26T04:44:34.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:44:34] ENGINE Serving on http://:::9283 2024-04-26T04:44:34.844 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:44:34] ENGINE Bus STARTED 2024-04-26T04:44:35.672 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:35 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:35.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:35 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:35.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:35 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:35.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:35 smithi171 ceph-mon[31139]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:35.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:35 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:44:35.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:35 smithi171 ceph-mon[31139]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:44:35.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:35 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:35.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:35 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:35.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:35 smithi171 ceph-mon[31139]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:44:35.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:35.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:35.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:35.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[27570]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:35.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:44:35.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[27570]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:44:35.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:35.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:35.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[27570]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:44:35.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:35.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:35.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:35.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[39130]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:35.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:44:35.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[39130]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:44:35.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:35.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:35.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:35 smithi156 ceph-mon[39130]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:44:36.736 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:36 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3739466244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:36.736 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:36 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:36.736 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:36 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:36.736 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:36 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:44:36.736 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:36 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:36.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:36 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3739466244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:36.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:36 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:36.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:36 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:36.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:36 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:44:36.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:36 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:36 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3739466244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:36 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:36 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:36 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:44:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:36 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:37.612 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:37 smithi171 ceph-mon[31139]: Upgrade: Updating mgr.x 2024-04-26T04:44:37.612 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:37 smithi171 ceph-mon[31139]: Deploying daemon mgr.x on smithi171 2024-04-26T04:44:37.612 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:37 smithi171 ceph-mon[31139]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:37.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:37 smithi156 ceph-mon[27570]: Upgrade: Updating mgr.x 2024-04-26T04:44:37.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:37 smithi156 ceph-mon[27570]: Deploying daemon mgr.x on smithi171 2024-04-26T04:44:37.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:37 smithi156 ceph-mon[27570]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:37.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:37 smithi156 ceph-mon[39130]: Upgrade: Updating mgr.x 2024-04-26T04:44:37.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:37 smithi156 ceph-mon[39130]: Deploying daemon mgr.x on smithi171 2024-04-26T04:44:37.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:37 smithi156 ceph-mon[39130]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:38.192 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:37 smithi171 systemd[1]: Stopping Ceph mgr.x for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:44:38.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:38 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3089021946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:38.499 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:38 smithi171 podman[79899]: 2024-04-26 04:44:38.19159878 +0000 UTC m=+0.141423390 container died 434a72ecc6a494c1da2650dac79c92e0f4f8031f49ac5ab7ac82037f03b2b0dc (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-26T04:44:38.744 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:38 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3089021946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:38.744 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:38 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3089021946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:38.932 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:38 smithi171 podman[79899]: 2024-04-26 04:44:38.532154757 +0000 UTC m=+0.481979363 container cleanup 434a72ecc6a494c1da2650dac79c92e0f4f8031f49ac5ab7ac82037f03b2b0dc (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x, GIT_CLEAN=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-26T04:44:38.932 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:38 smithi171 bash[79899]: ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x 2024-04-26T04:44:38.932 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:38 smithi171 podman[79918]: 2024-04-26 04:44:38.700253023 +0000 UTC m=+0.511633380 container remove 434a72ecc6a494c1da2650dac79c92e0f4f8031f49ac5ab7ac82037f03b2b0dc (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-26T04:44:38.932 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:38 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:44:39.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:38 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-26T04:44:39.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:38 smithi171 systemd[1]: Stopped Ceph mgr.x for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:39.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:38 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.x.service: Consumed 26.789s CPU time. 2024-04-26T04:44:39.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:39 smithi171 systemd[1]: Starting Ceph mgr.x for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:44:39.620 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:39 smithi171 ceph-mon[31139]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:39.621 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:39 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 3. 2024-04-26T04:44:39.621 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:39 smithi171 systemd[1]: Stopped Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:39.621 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:39 smithi171 systemd[1]: Starting Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:44:39.621 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:39 smithi171 podman[80052]: 2024-04-26 04:44:39.271095969 +0000 UTC m=+0.018650968 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:44:39.621 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:39 smithi171 podman[80052]: 2024-04-26 04:44:39.38478377 +0000 UTC m=+0.132338769 container create 9ec4d348d704a8af1fb289e626d003b972b022f50119a3d4d7a6d9844c6f4001 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, GIT_CLEAN=True) 2024-04-26T04:44:39.621 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:39 smithi171 podman[80052]: 2024-04-26 04:44:39.620635814 +0000 UTC m=+0.368190813 container init 9ec4d348d704a8af1fb289e626d003b972b022f50119a3d4d7a6d9844c6f4001 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:44:39.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:39 smithi156 ceph-mon[27570]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:39.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:39 smithi156 ceph-mon[39130]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:39.879 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:39 smithi171 podman[80052]: 2024-04-26 04:44:39.627171703 +0000 UTC m=+0.374726703 container start 9ec4d348d704a8af1fb289e626d003b972b022f50119a3d4d7a6d9844c6f4001 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-26T04:44:39.880 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:39 smithi171 ceph-mgr[80119]: -- 172.21.15.171:0/3102326666 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564be82ac5a0 con 0x564be8281000 2024-04-26T04:44:39.880 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:39 smithi171 bash[80052]: 9ec4d348d704a8af1fb289e626d003b972b022f50119a3d4d7a6d9844c6f4001 2024-04-26T04:44:39.880 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:39 smithi171 systemd[1]: Started Ceph mgr.x for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:39.880 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:39 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:39.797+0000 7f49bbdd8200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:44:40.132 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:39 smithi171 podman[80172]: 2024-04-26 04:44:39.781546292 +0000 UTC m=+0.019331058 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:44:40.133 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:40 smithi171 podman[80172]: 2024-04-26 04:44:40.030469853 +0000 UTC m=+0.268254607 container create e5a46620fa6b7d22c949ceebcf5196dd33fd1dff34abb040b7b8d181b93bd690 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, version=8.5, name=ubi8, io.openshift.expose-services=, release=239.1651231664, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, vendor=Red Hat, Inc., maintainer=Paul Cuzner , distribution-scope=public, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container) 2024-04-26T04:44:40.133 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:39 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:39.991+0000 7f49bbdd8200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:44:40.133 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:40.103+0000 7f49bbdd8200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:44:40.499 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:40.466+0000 7f49bbdd8200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:44:40.500 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:40 smithi171 podman[80172]: 2024-04-26 04:44:40.22400078 +0000 UTC m=+0.461785534 container init e5a46620fa6b7d22c949ceebcf5196dd33fd1dff34abb040b7b8d181b93bd690 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, maintainer=Paul Cuzner , distribution-scope=public, release=239.1651231664, vcs-type=git, io.buildah.version=1.32.0, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, name=ubi8) 2024-04-26T04:44:40.500 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:40 smithi171 podman[80172]: 2024-04-26 04:44:40.231415476 +0000 UTC m=+0.469200232 container start e5a46620fa6b7d22c949ceebcf5196dd33fd1dff34abb040b7b8d181b93bd690 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, vendor=Red Hat, Inc., io.openshift.expose-services=, release=239.1651231664, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , distribution-scope=public, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, name=ubi8, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-26T04:44:40.500 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:40 smithi171 bash[80172]: e5a46620fa6b7d22c949ceebcf5196dd33fd1dff34abb040b7b8d181b93bd690 2024-04-26T04:44:40.500 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:40 smithi171 systemd[1]: Started Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:40.500 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[80258]: logger=settings t=2024-04-26T04:44:40.402826774Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:44:40.811 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:40.553+0000 7f49bbdd8200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:44:41.069 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:40 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:41.069 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:40 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:41.069 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:40 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:41.070 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:40 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:41.070 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:40 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:41.070 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:40 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/79681214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:41.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:40 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:41.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:40 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:41.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:40 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:41.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:40 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:41.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:40 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:41.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:40 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/79681214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:41.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:40 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:41.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:40 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:41.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:40 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:41.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:40 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:41.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:40 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:41.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:40 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/79681214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:41.430 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:41 smithi171 podman[80336]: 2024-04-26 04:44:41.069186582 +0000 UTC m=+0.657963740 container remove e5a46620fa6b7d22c949ceebcf5196dd33fd1dff34abb040b7b8d181b93bd690 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, description=Ceph Grafana Container, distribution-scope=public) 2024-04-26T04:44:41.430 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:41 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:44:41.749 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:41 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:44:42.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:41 smithi156 ceph-mon[27570]: pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:42.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:41 smithi156 ceph-mon[39130]: pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:42.194 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:41 smithi171 ceph-mon[31139]: pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:42.194 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:41 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:41.829+0000 7f49bbdd8200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:44:42.194 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:42.016+0000 7f49bbdd8200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:44:42.499 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:42.193+0000 7f49bbdd8200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:44:42.833 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:42.502+0000 7f49bbdd8200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:44:42.833 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:42.598+0000 7f49bbdd8200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:44:43.180 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:42.831+0000 7f49bbdd8200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:44:43.180 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:43 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:43.039+0000 7f49bbdd8200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:44:43.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:43 smithi171 ceph-mon[31139]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:43.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:43 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/794843499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:44.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:43 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:43.673+0000 7f49bbdd8200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:44:44.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:43 smithi156 ceph-mon[27570]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:44.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:43 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/794843499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:44.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:43 smithi156 ceph-mon[39130]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:44.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:43 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/794843499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:44.425 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:44 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:44.139+0000 7f49bbdd8200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:44:44.425 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:44 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:44.244+0000 7f49bbdd8200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:44:44.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:44 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:44.460+0000 7f49bbdd8200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:44:45.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:44:44] "GET /metrics HTTP/1.1" 200 33793 "" "Prometheus/2.43.0" 2024-04-26T04:44:45.385 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:45.017+0000 7f49bbdd8200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:44:45.385 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:45.118+0000 7f49bbdd8200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:44:45.638 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:45.383+0000 7f49bbdd8200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:44:45.638 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:45.530+0000 7f49bbdd8200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:44:45.638 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:45.635+0000 7f49bbdd8200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:44:45.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:45 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:45.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:45 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:45.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:45 smithi171 ceph-mon[31139]: pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:46.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:45 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2096431319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:46.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:45.761+0000 7f49bbdd8200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:44:46.000 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:45.864+0000 7f49bbdd8200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:44:46.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:45 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:46.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:45 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:46.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:45 smithi156 ceph-mon[39130]: pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:46.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:45 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2096431319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:46.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:45 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:46.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:45 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:46.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:45 smithi156 ceph-mon[27570]: pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:46.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:45 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/2096431319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:46.749 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:46 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:46.742+0000 7f49bbdd8200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:44:47.083 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:46 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:47.083 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:46 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:47.083 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:46 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:46.974+0000 7f49bbdd8200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:44:47.083 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:46 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: [26/Apr/2024:04:44:46] ENGINE Bus STARTING 2024-04-26T04:44:47.083 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:47 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: CherryPy Checker: 2024-04-26T04:44:47.084 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:47 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: The Application mounted at '' has an empty config. 2024-04-26T04:44:47.084 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:47 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:47.084 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:47 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: [26/Apr/2024:04:44:47] ENGINE Serving on http://:::9283 2024-04-26T04:44:47.084 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:47 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: [26/Apr/2024:04:44:47] ENGINE Bus STARTED 2024-04-26T04:44:47.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:46 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:47.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:46 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:47.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:46 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:47.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:46 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:47.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:47 smithi171 ceph-mon[31139]: pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:48.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:47 smithi171 ceph-mon[31139]: Standby manager daemon x restarted 2024-04-26T04:44:48.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:47 smithi171 ceph-mon[31139]: Standby manager daemon x started 2024-04-26T04:44:48.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:47 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/866331090' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:44:48.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:47 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/866331090' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:44:48.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:47 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/866331090' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:44:48.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:47 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/866331090' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:44:48.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:47 smithi156 ceph-mon[27570]: pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:48.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:47 smithi156 ceph-mon[27570]: Standby manager daemon x restarted 2024-04-26T04:44:48.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:47 smithi156 ceph-mon[27570]: Standby manager daemon x started 2024-04-26T04:44:48.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:47 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/866331090' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:44:48.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:47 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/866331090' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:44:48.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:47 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/866331090' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:44:48.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:47 smithi156 ceph-mon[27570]: from='mgr.? 172.21.15.171:0/866331090' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:44:48.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:47 smithi156 ceph-mon[39130]: pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:48.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:47 smithi156 ceph-mon[39130]: Standby manager daemon x restarted 2024-04-26T04:44:48.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:47 smithi156 ceph-mon[39130]: Standby manager daemon x started 2024-04-26T04:44:48.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:47 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/866331090' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:44:48.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:47 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/866331090' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:44:48.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:47 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/866331090' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:44:48.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:47 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/866331090' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:44:48.831 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:48 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:44:49.141 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:48 smithi171 ceph-mon[31139]: mgrmap e33: y(active, since 29s), standbys: x 2024-04-26T04:44:49.141 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:48 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3201812129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:49.141 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:48 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:49.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:48 smithi156 ceph-mon[27570]: mgrmap e33: y(active, since 29s), standbys: x 2024-04-26T04:44:49.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:48 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3201812129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:49.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:48 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:49.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:48 smithi156 ceph-mon[39130]: mgrmap e33: y(active, since 29s), standbys: x 2024-04-26T04:44:49.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:48 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3201812129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:49.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:48 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:50.244 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[27570]: pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:50.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:50.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:50.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:50.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:44:50.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:50.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:50.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:44:50.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:50.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:44:50.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:44:50.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T04:44:50.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:44:50.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:44:50.245 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T04:44:50.246 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:44:50.246 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T04:44:50.246 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[39130]: pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:50.246 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:50.246 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:50.246 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:50.246 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:44:50.246 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:50.246 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:50.246 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:44:50.246 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:50.246 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:44:50.247 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:44:50.247 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T04:44:50.247 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:44:50.247 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:44:50.247 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T04:44:50.247 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:44:50.247 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:49 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T04:44:50.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:49 smithi171 ceph-mon[31139]: pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:50.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:49 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:50.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:49 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:50.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:49 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:49 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:44:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:49 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:49 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:49 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:44:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:49 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:49 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:44:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:49 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:44:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:49 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T04:44:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:49 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:44:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:49 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:44:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:49 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T04:44:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:49 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:44:50.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:49 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T04:44:51.082 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:50 smithi156 ceph-mon[27570]: pgmap v20: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:51.082 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:50 smithi156 ceph-mon[27570]: Upgrade: Setting container_image for all mgr 2024-04-26T04:44:51.082 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:50 smithi156 ceph-mon[27570]: Upgrade: It appears safe to stop mon.a 2024-04-26T04:44:51.082 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:50 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3043239657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:51.082 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:50 smithi156 ceph-mon[39130]: pgmap v20: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:51.082 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:50 smithi156 ceph-mon[39130]: Upgrade: Setting container_image for all mgr 2024-04-26T04:44:51.082 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:50 smithi156 ceph-mon[39130]: Upgrade: It appears safe to stop mon.a 2024-04-26T04:44:51.083 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:50 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3043239657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:51.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:50 smithi171 ceph-mon[31139]: pgmap v20: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:51.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:50 smithi171 ceph-mon[31139]: Upgrade: Setting container_image for all mgr 2024-04-26T04:44:51.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:50 smithi171 ceph-mon[31139]: Upgrade: It appears safe to stop mon.a 2024-04-26T04:44:51.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:50 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3043239657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:52.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:51 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 4. 2024-04-26T04:44:52.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:51 smithi171 systemd[1]: Stopped Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:52.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:51 smithi171 systemd[1]: Starting Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:44:52.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:51 smithi171 podman[83644]: 2024-04-26 04:44:51.982948602 +0000 UTC m=+0.105911564 container create 7864853939d3827fd91133ec1868034dccb8f10250da401e0847516d7e3c6f2c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, io.openshift.expose-services=, io.buildah.version=1.32.0, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, distribution-scope=public, maintainer=Paul Cuzner , version=8.5, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-26T04:44:52.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:51 smithi171 podman[83644]: 2024-04-26 04:44:51.894852046 +0000 UTC m=+0.017814978 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:44:52.331 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:52 smithi156 ceph-mon[27570]: Upgrade: Updating mon.a 2024-04-26T04:44:52.331 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:52 smithi156 ceph-mon[27570]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:52.331 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:52 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:44:52.331 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:52 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:44:52.331 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:52 smithi156 ceph-mon[27570]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:52.331 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:52 smithi156 ceph-mon[27570]: Deploying daemon mon.a on smithi156 2024-04-26T04:44:52.331 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:52 smithi156 ceph-mon[27570]: pgmap v21: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:52.332 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:52 smithi156 ceph-mon[39130]: Upgrade: Updating mon.a 2024-04-26T04:44:52.332 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:52 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:52.332 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:52 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:44:52.332 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:52 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:44:52.332 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:52 smithi156 ceph-mon[39130]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:52.332 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:52 smithi156 ceph-mon[39130]: Deploying daemon mon.a on smithi156 2024-04-26T04:44:52.332 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:52 smithi156 ceph-mon[39130]: pgmap v21: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:52.352 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:52 smithi171 ceph-mon[31139]: Upgrade: Updating mon.a 2024-04-26T04:44:52.352 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:52 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='mgr.y' 2024-04-26T04:44:52.352 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:52 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:44:52.352 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:52 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:44:52.352 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:52 smithi171 ceph-mon[31139]: from='mgr.25070 172.21.15.156:0/4035771915' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:52.352 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:52 smithi171 ceph-mon[31139]: Deploying daemon mon.a on smithi156 2024-04-26T04:44:52.352 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:52 smithi171 ceph-mon[31139]: pgmap v21: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:52.353 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:52 smithi171 podman[83644]: 2024-04-26 04:44:52.243480369 +0000 UTC m=+0.366443288 container init 7864853939d3827fd91133ec1868034dccb8f10250da401e0847516d7e3c6f2c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, release=239.1651231664, io.buildah.version=1.32.0, description=Ceph Grafana Container, maintainer=Paul Cuzner , version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, architecture=x86_64, vcs-type=git, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container) 2024-04-26T04:44:52.353 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:52 smithi171 podman[83644]: 2024-04-26 04:44:52.248931794 +0000 UTC m=+0.371894714 container start 7864853939d3827fd91133ec1868034dccb8f10250da401e0847516d7e3c6f2c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, io.buildah.version=1.32.0, io.openshift.expose-services=, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, version=8.5, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664) 2024-04-26T04:44:52.738 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:52 smithi171 bash[83644]: 7864853939d3827fd91133ec1868034dccb8f10250da401e0847516d7e3c6f2c 2024-04-26T04:44:52.738 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:52 smithi171 systemd[1]: Started Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:52.738 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:52 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[83659]: logger=settings t=2024-04-26T04:44:52.409527232Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:44:52.738 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:52 smithi171 podman[83695]: 2024-04-26 04:44:52.435570786 +0000 UTC m=+0.018895851 container died 7864853939d3827fd91133ec1868034dccb8f10250da401e0847516d7e3c6f2c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, description=Ceph Grafana Container, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, name=ubi8, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:44:53.259 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:53 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3752380320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:53.260 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:52 smithi156 systemd[1]: Stopping Ceph mon.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:44:53.260 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:53 smithi156 ceph-mon[27570]: from='client.? 172.21.15.156:0/3752380320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:53.260 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:53 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a[27547]: 2024-04-26T04:44:53.236+0000 7f8b464bf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:44:53.260 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:53 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a[27547]: 2024-04-26T04:44:53.236+0000 7f8b464bf700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-26T04:44:53.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:53 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3752380320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:53.500 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:53 smithi171 podman[83695]: 2024-04-26 04:44:53.013824006 +0000 UTC m=+0.597149116 container remove 7864853939d3827fd91133ec1868034dccb8f10250da401e0847516d7e3c6f2c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, architecture=x86_64, description=Ceph Grafana Container, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , vcs-type=git, vendor=Red Hat, Inc.) 2024-04-26T04:44:53.500 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:53 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:44:53.500 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:44:53 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:44:53.553 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:53 smithi156 podman[98549]: 2024-04-26 04:44:53.258675114 +0000 UTC m=+0.119612726 container died 3d899f93d0ef43f8f8e9bdc515dcd625816d215720310e33a0e6709fa445ba00 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True) 2024-04-26T04:44:53.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:53 smithi156 podman[98549]: 2024-04-26 04:44:53.752122552 +0000 UTC m=+0.613060213 container cleanup 3d899f93d0ef43f8f8e9bdc515dcd625816d215720310e33a0e6709fa445ba00 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_CLEAN=True, RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-26T04:44:53.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:53 smithi156 bash[98549]: ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a 2024-04-26T04:44:54.299 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:53 smithi156 podman[98569]: 2024-04-26 04:44:53.978852384 +0000 UTC m=+0.722573651 container remove 3d899f93d0ef43f8f8e9bdc515dcd625816d215720310e33a0e6709fa445ba00 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:44:54.299 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-26T04:44:54.300 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 systemd[1]: Stopped Ceph mon.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:54.300 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.a.service: Consumed 12.152s CPU time. 2024-04-26T04:44:54.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 systemd[1]: Starting Ceph mon.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:44:54.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 podman[98705]: 2024-04-26 04:44:54.41645448 +0000 UTC m=+0.018485849 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:44:54.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 podman[98705]: 2024-04-26 04:44:54.546904079 +0000 UTC m=+0.148935427 container create b94e4b5539a16e33be567f6e470d89308a5a962112f4bb5960609def847706ba (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS) 2024-04-26T04:44:55.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:44:54] "GET /metrics HTTP/1.1" 200 33787 "" "Prometheus/2.43.0" 2024-04-26T04:44:55.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 podman[98705]: 2024-04-26 04:44:54.783239899 +0000 UTC m=+0.385271258 container init b94e4b5539a16e33be567f6e470d89308a5a962112f4bb5960609def847706ba (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb) 2024-04-26T04:44:55.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 podman[98705]: 2024-04-26 04:44:54.789093835 +0000 UTC m=+0.391125182 container start b94e4b5539a16e33be567f6e470d89308a5a962112f4bb5960609def847706ba (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, ceph=True) 2024-04-26T04:44:55.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:44:55.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 7 2024-04-26T04:44:55.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: pidfile_write: ignore empty --pid-file 2024-04-26T04:44:55.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: load: jerasure load: lrc 2024-04-26T04:44:55.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: RocksDB version: 7.9.2 2024-04-26T04:44:55.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Git sha 0 2024-04-26T04:44:55.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-26T04:44:55.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: DB SUMMARY 2024-04-26T04:44:55.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: DB Session ID: P4H5E1YIURQWILV0I8ZK 2024-04-26T04:44:55.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:44:55.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:44:55.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: MANIFEST file: MANIFEST-000015 size: 972 Bytes 2024-04-26T04:44:55.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000030.sst 2024-04-26T04:44:55.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000028.log size: 2737490 ; 2024-04-26T04:44:55.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:44:55.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:44:55.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:44:55.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T04:44:55.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:44:55.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T04:44:55.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.env: 0x560069ce8c60 2024-04-26T04:44:55.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T04:44:55.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.info_log: 0x56006c453aa0 2024-04-26T04:44:55.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:44:55.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.statistics: (nil) 2024-04-26T04:44:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.use_fsync: 0 2024-04-26T04:44:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:44:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:44:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:44:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:44:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:44:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:44:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:44:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:44:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:44:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:44:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:44:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.db_log_dir: 2024-04-26T04:44:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.wal_dir: 2024-04-26T04:44:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:44:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:44:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:44:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:44:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:44:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:44:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:44:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:44:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.write_buffer_manager: 0x56006c426820 2024-04-26T04:44:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:44:55.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:44:55.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:44:55.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:44:55.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:44:55.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:44:55.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:44:55.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:44:55.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.unordered_write: 0 2024-04-26T04:44:55.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:44:55.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:44:55.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:44:55.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:44:55.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.row_cache: None 2024-04-26T04:44:55.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.wal_filter: None 2024-04-26T04:44:55.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:44:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:44:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:44:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:44:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.wal_compression: 0 2024-04-26T04:44:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:44:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:44:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:44:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:44:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:44:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:44:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:44:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:44:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:44:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:44:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:44:55.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T04:44:55.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:44:55.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:44:55.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:44:55.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:44:55.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:44:55.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:44:55.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:44:55.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:44:55.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:44:55.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:44:55.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:44:55.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_open_files: -1 2024-04-26T04:44:55.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:44:55.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:44:55.101 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:44:55.101 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:44:55.101 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:44:55.101 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Compression algorithms supported: 2024-04-26T04:44:55.101 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:44:55.101 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: kZSTD supported: 0 2024-04-26T04:44:55.101 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:44:55.101 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:44:55.101 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:44:55.101 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:44:55.101 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:44:55.101 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:44:55.102 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:44:55.102 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T04:44:55.102 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-04-26T04:44:55.102 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T04:44:55.102 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:44:55.102 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.merge_operator: 2024-04-26T04:44:55.102 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compaction_filter: None 2024-04-26T04:44:55.102 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:44:55.102 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:44:55.102 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:44:55.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:44:55.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56006c453ba0) 2024-04-26T04:44:55.103 INFO:journalctl@ceph.mon.a.smithi156.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:44:55.103 INFO:journalctl@ceph.mon.a.smithi156.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:44:55.103 INFO:journalctl@ceph.mon.a.smithi156.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:44:55.103 INFO:journalctl@ceph.mon.a.smithi156.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:44:55.103 INFO:journalctl@ceph.mon.a.smithi156.stdout: index_type: 0 2024-04-26T04:44:55.103 INFO:journalctl@ceph.mon.a.smithi156.stdout: data_block_index_type: 0 2024-04-26T04:44:55.103 INFO:journalctl@ceph.mon.a.smithi156.stdout: index_shortening: 1 2024-04-26T04:44:55.103 INFO:journalctl@ceph.mon.a.smithi156.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:44:55.103 INFO:journalctl@ceph.mon.a.smithi156.stdout: checksum: 4 2024-04-26T04:44:55.103 INFO:journalctl@ceph.mon.a.smithi156.stdout: no_block_cache: 0 2024-04-26T04:44:55.103 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_cache: 0x56006c457090 2024-04-26T04:44:55.103 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:44:55.103 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_cache_options: 2024-04-26T04:44:55.104 INFO:journalctl@ceph.mon.a.smithi156.stdout: capacity : 536870912 2024-04-26T04:44:55.104 INFO:journalctl@ceph.mon.a.smithi156.stdout: num_shard_bits : 4 2024-04-26T04:44:55.104 INFO:journalctl@ceph.mon.a.smithi156.stdout: strict_capacity_limit : 0 2024-04-26T04:44:55.104 INFO:journalctl@ceph.mon.a.smithi156.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:44:55.104 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_cache_compressed: (nil) 2024-04-26T04:44:55.104 INFO:journalctl@ceph.mon.a.smithi156.stdout: persistent_cache: (nil) 2024-04-26T04:44:55.104 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_size: 4096 2024-04-26T04:44:55.104 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_size_deviation: 10 2024-04-26T04:44:55.104 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_restart_interval: 16 2024-04-26T04:44:55.104 INFO:journalctl@ceph.mon.a.smithi156.stdout: index_block_restart_interval: 1 2024-04-26T04:44:55.104 INFO:journalctl@ceph.mon.a.smithi156.stdout: metadata_block_size: 4096 2024-04-26T04:44:55.104 INFO:journalctl@ceph.mon.a.smithi156.stdout: partition_filters: 0 2024-04-26T04:44:55.104 INFO:journalctl@ceph.mon.a.smithi156.stdout: use_delta_encoding: 1 2024-04-26T04:44:55.105 INFO:journalctl@ceph.mon.a.smithi156.stdout: filter_policy: bloomfilter 2024-04-26T04:44:55.105 INFO:journalctl@ceph.mon.a.smithi156.stdout: whole_key_filtering: 1 2024-04-26T04:44:55.105 INFO:journalctl@ceph.mon.a.smithi156.stdout: verify_compression: 0 2024-04-26T04:44:55.105 INFO:journalctl@ceph.mon.a.smithi156.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:44:55.105 INFO:journalctl@ceph.mon.a.smithi156.stdout: format_version: 5 2024-04-26T04:44:55.105 INFO:journalctl@ceph.mon.a.smithi156.stdout: enable_index_compression: 1 2024-04-26T04:44:55.105 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_align: 0 2024-04-26T04:44:55.105 INFO:journalctl@ceph.mon.a.smithi156.stdout: max_auto_readahead_size: 262144 2024-04-26T04:44:55.105 INFO:journalctl@ceph.mon.a.smithi156.stdout: prepopulate_block_cache: 0 2024-04-26T04:44:55.105 INFO:journalctl@ceph.mon.a.smithi156.stdout: initial_auto_readahead_size: 8192 2024-04-26T04:44:55.105 INFO:journalctl@ceph.mon.a.smithi156.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T04:44:55.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:44:55.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:44:55.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compression: NoCompression 2024-04-26T04:44:55.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:44:55.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:44:55.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:44:55.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.num_levels: 7 2024-04-26T04:44:55.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:44:55.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:44:55.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:44:55.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:44:55.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:44:55.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:44:55.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:44:55.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:44:55.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:44:55.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:44:55.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:44:55.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:44:55.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:44:55.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:44:55.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:44:55.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:44:55.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:44:55.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:44:55.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:44:55.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:44:55.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:44:55.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:44:55.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:44:55.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:44:55.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:44:55.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:44:55.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:44:55.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:44:55.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:44:55.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:44:55.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:44:55.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:44:55.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:44:55.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:44:55.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:44:55.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:44:55.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:44:55.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:44:55.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T04:44:55.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T04:44:55.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:44:55.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:44:55.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:44:55.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:44:55.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:44:55.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:44:55.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:44:55.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:44:55.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:44:55.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:44:55.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:44:55.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:44:55.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:44:55.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:44:55.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:44:55.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:44:55.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:44:55.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:44:55.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:44:55.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:44:55.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:44:55.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:44:55.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:44:55.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:44:55.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:44:55.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.ttl: 2592000 2024-04-26T04:44:55.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:44:55.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T04:44:55.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T04:44:55.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:44:55.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:44:55.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:44:55.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:44:55.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:44:55.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:44:55.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T04:44:55.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T04:44:55.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T04:44:55.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T04:44:55.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-04-26T04:44:55.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:44:55.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 10542, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:44:55.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-04-26T04:44:55.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 11e49592-8486-4c7b-abce-1cd7cefe9242 2024-04-26T04:44:55.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106694818829, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-04-26T04:44:55.115 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-04-26T04:44:55.115 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106694826991, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2505859, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 10543, "largest_seqno": 11813, "table_properties": {"data_size": 2499474, "index_size": 3830, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1541, "raw_key_size": 14288, "raw_average_key_size": 24, "raw_value_size": 2486576, "raw_average_value_size": 4193, "num_data_blocks": 175, "num_entries": 593, "num_filter_entries": 593, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714106694, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "11e49592-8486-4c7b-abce-1cd7cefe9242", "db_session_id": "P4H5E1YIURQWILV0I8ZK", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-04-26T04:44:55.115 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106694827086, "job": 1, "event": "recovery_finished"} 2024-04-26T04:44:55.115 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-04-26T04:44:55.115 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:44:55.115 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:44:55.115 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56006c548000 2024-04-26T04:44:55.115 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: DB pointer 0x56006c532000 2024-04-26T04:44:55.115 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T04:44:55.115 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T04:44:55.115 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** DB Stats ** 2024-04-26T04:44:55.115 INFO:journalctl@ceph.mon.a.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:44:55.115 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:44:55.115 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:44:55.116 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:44:55.116 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:44:55.116 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:44:55.116 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:44:55.116 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-04-26T04:44:55.116 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** Compaction Stats [default] ** 2024-04-26T04:44:55.116 INFO:journalctl@ceph.mon.a.smithi156.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:44:55.116 INFO:journalctl@ceph.mon.a.smithi156.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T04:44:55.116 INFO:journalctl@ceph.mon.a.smithi156.stdout: L0 1/0 2.39 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 565.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T04:44:55.116 INFO:journalctl@ceph.mon.a.smithi156.stdout: L6 1/0 6.24 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T04:44:55.116 INFO:journalctl@ceph.mon.a.smithi156.stdout: Sum 2/0 8.63 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 565.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T04:44:55.116 INFO:journalctl@ceph.mon.a.smithi156.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 565.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T04:44:55.116 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-04-26T04:44:55.116 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** Compaction Stats [default] ** 2024-04-26T04:44:55.116 INFO:journalctl@ceph.mon.a.smithi156.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:44:55.117 INFO:journalctl@ceph.mon.a.smithi156.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:44:55.117 INFO:journalctl@ceph.mon.a.smithi156.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 565.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T04:44:55.117 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-04-26T04:44:55.117 INFO:journalctl@ceph.mon.a.smithi156.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T04:44:55.117 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-04-26T04:44:55.117 INFO:journalctl@ceph.mon.a.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:44:55.117 INFO:journalctl@ceph.mon.a.smithi156.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-26T04:44:55.117 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:44:55.117 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:44:55.117 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:44:55.117 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:44:55.117 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative compaction: 0.00 GB write, 226.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:44:55.117 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval compaction: 0.00 GB write, 226.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:44:55.117 INFO:journalctl@ceph.mon.a.smithi156.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:44:55.118 INFO:journalctl@ceph.mon.a.smithi156.stdout: Block cache BinnedLRUCache@0x56006c457090#7 capacity: 512.00 MB usage: 5.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-04-26T04:44:55.118 INFO:journalctl@ceph.mon.a.smithi156.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-04-26T04:44:55.118 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-04-26T04:44:55.118 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:44:55.118 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: starting mon.a rank 0 at public addrs [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] at bind addrs [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:44:55.118 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: mon.a@-1(???) e3 preinit fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:44:55.118 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: mon.a@-1(???).mds e9 new map 2024-04-26T04:44:55.118 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: mon.a@-1(???).mds e9 print_map 2024-04-26T04:44:55.118 INFO:journalctl@ceph.mon.a.smithi156.stdout: e9 2024-04-26T04:44:55.118 INFO:journalctl@ceph.mon.a.smithi156.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:44:55.118 INFO:journalctl@ceph.mon.a.smithi156.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:44:55.118 INFO:journalctl@ceph.mon.a.smithi156.stdout: legacy client fscid: 1 2024-04-26T04:44:55.118 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-04-26T04:44:55.119 INFO:journalctl@ceph.mon.a.smithi156.stdout: Filesystem 'foo' (1) 2024-04-26T04:44:55.119 INFO:journalctl@ceph.mon.a.smithi156.stdout: fs_name foo 2024-04-26T04:44:55.119 INFO:journalctl@ceph.mon.a.smithi156.stdout: epoch 8 2024-04-26T04:44:55.119 INFO:journalctl@ceph.mon.a.smithi156.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-26T04:44:55.119 INFO:journalctl@ceph.mon.a.smithi156.stdout: created 2024-04-26T04:40:44.280108+0000 2024-04-26T04:44:55.119 INFO:journalctl@ceph.mon.a.smithi156.stdout: modified 2024-04-26T04:40:54.847092+0000 2024-04-26T04:44:55.121 INFO:journalctl@ceph.mon.a.smithi156.stdout: tableserver 0 2024-04-26T04:44:55.121 INFO:journalctl@ceph.mon.a.smithi156.stdout: root 0 2024-04-26T04:44:55.121 INFO:journalctl@ceph.mon.a.smithi156.stdout: session_timeout 60 2024-04-26T04:44:55.121 INFO:journalctl@ceph.mon.a.smithi156.stdout: session_autoclose 300 2024-04-26T04:44:55.121 INFO:journalctl@ceph.mon.a.smithi156.stdout: max_file_size 1099511627776 2024-04-26T04:44:55.121 INFO:journalctl@ceph.mon.a.smithi156.stdout: max_xattr_size 65536 2024-04-26T04:44:55.121 INFO:journalctl@ceph.mon.a.smithi156.stdout: required_client_features {} 2024-04-26T04:44:55.121 INFO:journalctl@ceph.mon.a.smithi156.stdout: last_failure 0 2024-04-26T04:44:55.121 INFO:journalctl@ceph.mon.a.smithi156.stdout: last_failure_osd_epoch 0 2024-04-26T04:44:55.122 INFO:journalctl@ceph.mon.a.smithi156.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:44:55.122 INFO:journalctl@ceph.mon.a.smithi156.stdout: max_mds 1 2024-04-26T04:44:55.122 INFO:journalctl@ceph.mon.a.smithi156.stdout: in 0 2024-04-26T04:44:55.122 INFO:journalctl@ceph.mon.a.smithi156.stdout: up {0=14637} 2024-04-26T04:44:55.122 INFO:journalctl@ceph.mon.a.smithi156.stdout: failed 2024-04-26T04:44:55.122 INFO:journalctl@ceph.mon.a.smithi156.stdout: damaged 2024-04-26T04:44:55.122 INFO:journalctl@ceph.mon.a.smithi156.stdout: stopped 2024-04-26T04:44:55.122 INFO:journalctl@ceph.mon.a.smithi156.stdout: data_pools [4] 2024-04-26T04:44:55.122 INFO:journalctl@ceph.mon.a.smithi156.stdout: metadata_pool 3 2024-04-26T04:44:55.122 INFO:journalctl@ceph.mon.a.smithi156.stdout: inline_data disabled 2024-04-26T04:44:55.122 INFO:journalctl@ceph.mon.a.smithi156.stdout: balancer 2024-04-26T04:44:55.122 INFO:journalctl@ceph.mon.a.smithi156.stdout: bal_rank_mask -1 2024-04-26T04:44:55.122 INFO:journalctl@ceph.mon.a.smithi156.stdout: standby_count_wanted 1 2024-04-26T04:44:55.122 INFO:journalctl@ceph.mon.a.smithi156.stdout: qdb_cluster leader: 0 members: 2024-04-26T04:44:55.123 INFO:journalctl@ceph.mon.a.smithi156.stdout: [mds.foo.smithi156.wyfzii{0:14637} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.156:6834/572371201,v1:172.21.15.156:6835/572371201] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:44:55.123 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-04-26T04:44:55.123 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-04-26T04:44:55.123 INFO:journalctl@ceph.mon.a.smithi156.stdout: Standby daemons: 2024-04-26T04:44:55.123 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-04-26T04:44:55.123 INFO:journalctl@ceph.mon.a.smithi156.stdout: [mds.foo.smithi171.kzjlqn{-1:24478} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.171:6832/1083563586,v1:172.21.15.171:6833/1083563586] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:44:55.123 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: mon.a@-1(???).osd e77 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T04:44:55.123 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: mon.a@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:44:55.123 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: mon.a@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:44:55.123 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: mon.a@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:44:55.123 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 ceph-mon[98743]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2024-04-26T04:44:55.123 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 bash[98705]: b94e4b5539a16e33be567f6e470d89308a5a962112f4bb5960609def847706ba 2024-04-26T04:44:55.123 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:54 smithi156 systemd[1]: Started Ceph mon.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:44:58.218 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ignoring --setuser ceph since I am not root 2024-04-26T04:44:58.218 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ignoring --setgroup ceph since I am not root 2024-04-26T04:44:58.218 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mgr[89844]: -- 172.21.15.156:0/2585476898 <== mon.0 v2:172.21.15.156:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55bb33a7e5a0 con 0x55bb33a52800 2024-04-26T04:44:58.218 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:58 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:58.033+0000 7f745ef80200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:44:58.218 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:58 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:58.217+0000 7f745ef80200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:44:58.219 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:58.219 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: pgmap v23: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:58.219 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/3584787094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:58.219 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: mon.a calling monitor election 2024-04-26T04:44:58.219 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/2643821715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:58.219 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:44:58.219 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: monmap epoch 3 2024-04-26T04:44:58.219 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:44:58.219 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: last_changed 2024-04-26T04:35:24.769972+0000 2024-04-26T04:44:58.219 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: created 2024-04-26T04:33:43.779990+0000 2024-04-26T04:44:58.219 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: min_mon_release 17 (quincy) 2024-04-26T04:44:58.219 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: election_strategy: 1 2024-04-26T04:44:58.219 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-04-26T04:44:58.219 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-04-26T04:44:58.220 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-04-26T04:44:58.220 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:44:58.220 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:44:58.220 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: mgrmap e33: y(active, since 39s), standbys: x 2024-04-26T04:44:58.220 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:44:58.220 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:44:58.220 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: daemon grafana.a on smithi171 is in unknown state 2024-04-26T04:44:58.220 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: from='mgr.25070 ' entity='' 2024-04-26T04:44:58.220 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[39130]: mgrmap e34: y(active, since 39s), standbys: x 2024-04-26T04:44:58.220 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:58.220 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: pgmap v23: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:58.220 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3584787094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:58.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: mon.a calling monitor election 2024-04-26T04:44:58.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2643821715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:58.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:44:58.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: monmap epoch 3 2024-04-26T04:44:58.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:44:58.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: last_changed 2024-04-26T04:35:24.769972+0000 2024-04-26T04:44:58.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: created 2024-04-26T04:33:43.779990+0000 2024-04-26T04:44:58.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: min_mon_release 17 (quincy) 2024-04-26T04:44:58.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: election_strategy: 1 2024-04-26T04:44:58.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-04-26T04:44:58.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-04-26T04:44:58.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-04-26T04:44:58.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:44:58.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:44:58.222 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: mgrmap e33: y(active, since 39s), standbys: x 2024-04-26T04:44:58.222 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:44:58.222 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:44:58.222 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: daemon grafana.a on smithi171 is in unknown state 2024-04-26T04:44:58.222 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: from='mgr.25070 ' entity='' 2024-04-26T04:44:58.222 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:44:57 smithi156 ceph-mon[98743]: mgrmap e34: y(active, since 39s), standbys: x 2024-04-26T04:44:58.249 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: ignoring --setuser ceph since I am not root 2024-04-26T04:44:58.249 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: ignoring --setgroup ceph since I am not root 2024-04-26T04:44:58.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mgr[80119]: -- 172.21.15.171:0/2197017666 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55999ac405a0 con 0x55999ac14800 2024-04-26T04:44:58.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:58 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:58.051+0000 7f95eedcb200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:44:58.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:58.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: pgmap v23: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:44:58.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3584787094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:58.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: mon.a calling monitor election 2024-04-26T04:44:58.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/2643821715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:58.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:44:58.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: monmap epoch 3 2024-04-26T04:44:58.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:44:58.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: last_changed 2024-04-26T04:35:24.769972+0000 2024-04-26T04:44:58.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: created 2024-04-26T04:33:43.779990+0000 2024-04-26T04:44:58.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: min_mon_release 17 (quincy) 2024-04-26T04:44:58.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: election_strategy: 1 2024-04-26T04:44:58.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-04-26T04:44:58.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-04-26T04:44:58.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-04-26T04:44:58.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:44:58.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:44:58.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: mgrmap e33: y(active, since 39s), standbys: x 2024-04-26T04:44:58.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:44:58.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:44:58.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: daemon grafana.a on smithi171 is in unknown state 2024-04-26T04:44:58.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: from='mgr.25070 ' entity='' 2024-04-26T04:44:58.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:44:57 smithi171 ceph-mon[31139]: mgrmap e34: y(active, since 39s), standbys: x 2024-04-26T04:44:58.593 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:58 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:58.319+0000 7f745ef80200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:44:58.659 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:58 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:58.254+0000 7f95eedcb200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:44:58.659 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:58 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:58.368+0000 7f95eedcb200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:44:58.999 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:58 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:58.676+0000 7f95eedcb200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:44:58.999 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:58 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:58.761+0000 7f95eedcb200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:44:59.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:44:58 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:44:58.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:44:59.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:58 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:58.626+0000 7f745ef80200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:44:59.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:58 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:58.711+0000 7f745ef80200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:45:00.086 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 1 2024-04-26T04:45:00.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:00 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/328801777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:00.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:44:59 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:44:59.947+0000 7f95eedcb200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:45:00.250 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:00.119+0000 7f95eedcb200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:45:00.277 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:44:59 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:44:59.919+0000 7f745ef80200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:45:00.277 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:00 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:00.098+0000 7f745ef80200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:45:00.278 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:00 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/328801777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:00.278 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:00 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/328801777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:00.574 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:00 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:00.276+0000 7f745ef80200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:45:00.575 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:00 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:00.573+0000 7f745ef80200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:45:00.591 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:00.295+0000 7f95eedcb200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:45:00.591 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:00.589+0000 7f95eedcb200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:45:00.830 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:00 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:00.668+0000 7f745ef80200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:45:00.832 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:45:00.893 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:00.684+0000 7f95eedcb200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:45:01.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:00 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:00.881+0000 7f745ef80200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:45:01.094 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:01.249 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:00 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:00.891+0000 7f95eedcb200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:45:01.249 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:01.086+0000 7f95eedcb200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:45:01.347 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:01 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:01.093+0000 7f745ef80200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:45:01.999 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:01 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:01.694+0000 7f95eedcb200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:45:02.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:01 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:01.729+0000 7f745ef80200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:45:02.432 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:02 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:02.124+0000 7f95eedcb200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:45:02.432 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:02 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:02.223+0000 7f95eedcb200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:45:02.432 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:02 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:02.430+0000 7f95eedcb200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:45:02.448 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:02 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:02.164+0000 7f745ef80200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:45:02.448 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:02 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:02.262+0000 7f745ef80200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:45:02.750 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:02 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:02.479+0000 7f745ef80200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:45:03.249 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:02 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:02.959+0000 7f95eedcb200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:45:03.249 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:03.052+0000 7f95eedcb200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:45:03.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:03 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:03.028+0000 7f745ef80200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:45:03.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:03 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:03.122+0000 7f745ef80200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:45:03.675 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:03 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 5. 2024-04-26T04:45:03.675 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:03 smithi171 systemd[1]: Stopped Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:45:03.675 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:03 smithi171 systemd[1]: Starting Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:45:03.675 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:03.299+0000 7f95eedcb200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:45:03.675 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:03.435+0000 7f95eedcb200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:45:03.675 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:03.534+0000 7f95eedcb200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:45:03.725 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:03 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:03.372+0000 7f745ef80200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:45:03.725 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:03 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:03.508+0000 7f745ef80200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:45:03.726 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:03 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:03.607+0000 7f745ef80200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:45:03.726 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:03 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:03.724+0000 7f745ef80200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:45:03.999 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:03.673+0000 7f95eedcb200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:45:03.999 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:03.772+0000 7f95eedcb200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:45:04.000 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:03 smithi171 podman[83884]: 2024-04-26 04:45:03.641518775 +0000 UTC m=+0.018943404 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:45:04.000 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:03 smithi171 podman[83884]: 2024-04-26 04:45:03.772632735 +0000 UTC m=+0.150057345 container create 0382a3be8f17f1f166f425f276ef2bd9bdbd487011b7d85dbc5856efd0b1aa91 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, description=Ceph Grafana Container, architecture=x86_64, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:45:04.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:03 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:03.821+0000 7f745ef80200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:45:04.499 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:04 smithi171 podman[83884]: 2024-04-26 04:45:04.024802281 +0000 UTC m=+0.402226894 container init 0382a3be8f17f1f166f425f276ef2bd9bdbd487011b7d85dbc5856efd0b1aa91 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, version=8.5, com.redhat.component=ubi8-container, name=ubi8, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, io.openshift.expose-services=, maintainer=Paul Cuzner , io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8) 2024-04-26T04:45:04.500 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:04 smithi171 podman[83884]: 2024-04-26 04:45:04.03033363 +0000 UTC m=+0.407758241 container start 0382a3be8f17f1f166f425f276ef2bd9bdbd487011b7d85dbc5856efd0b1aa91 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, io.openshift.expose-services=, version=8.5, description=Ceph Grafana Container, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, architecture=x86_64, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-26T04:45:04.500 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:04 smithi171 bash[83884]: 0382a3be8f17f1f166f425f276ef2bd9bdbd487011b7d85dbc5856efd0b1aa91 2024-04-26T04:45:04.500 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:04 smithi171 systemd[1]: Started Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:45:04.500 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[83899]: logger=settings t=2024-04-26T04:45:04.19928272Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:45:04.500 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:04 smithi171 podman[83937]: 2024-04-26 04:45:04.224985331 +0000 UTC m=+0.018580497 container died 0382a3be8f17f1f166f425f276ef2bd9bdbd487011b7d85dbc5856efd0b1aa91 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., io.buildah.version=1.32.0, description=Ceph Grafana Container, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.component=ubi8-container, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, distribution-scope=public, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=) 2024-04-26T04:45:04.865 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:04.639+0000 7f95eedcb200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:45:04.918 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:04.646+0000 7f745ef80200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:45:04.918 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:04.873+0000 7f745ef80200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:45:05.125 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:04 smithi171 podman[83937]: 2024-04-26 04:45:04.878802166 +0000 UTC m=+0.672397331 container remove 0382a3be8f17f1f166f425f276ef2bd9bdbd487011b7d85dbc5856efd0b1aa91 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.openshift.expose-services=, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, description=Ceph Grafana Container, io.buildah.version=1.32.0, architecture=x86_64, maintainer=Paul Cuzner , name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container) 2024-04-26T04:45:05.125 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:04 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:45:05.126 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:04.863+0000 7f95eedcb200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:45:05.126 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: [26/Apr/2024:04:45:04] ENGINE Bus STARTING 2024-04-26T04:45:05.126 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: CherryPy Checker: 2024-04-26T04:45:05.126 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: The Application mounted at '' has an empty config. 2024-04-26T04:45:05.126 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: 2024-04-26T04:45:05.126 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: [26/Apr/2024:04:45:04] ENGINE Serving on http://:::9283 2024-04-26T04:45:05.126 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x[80066]: [26/Apr/2024:04:45:04] ENGINE Bus STARTED 2024-04-26T04:45:05.126 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: Standby manager daemon x restarted 2024-04-26T04:45:05.126 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: Standby manager daemon x started 2024-04-26T04:45:05.126 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/4026315501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:45:05.127 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/4026315501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:45:05.127 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/4026315501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:45:05.127 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.? 172.21.15.171:0/4026315501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:45:05.127 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: Active manager daemon y restarted 2024-04-26T04:45:05.127 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: Activating manager daemon y 2024-04-26T04:45:05.127 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:45:05.127 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: mgrmap e35: y(active, starting, since 0.0068191s), standbys: x 2024-04-26T04:45:05.127 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:45:05.127 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:45:05.127 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:45:05.127 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.wyfzii"}]: dispatch 2024-04-26T04:45:05.127 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.kzjlqn"}]: dispatch 2024-04-26T04:45:05.127 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:45:05.127 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:45:05.128 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:45:05.128 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:45:05.128 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:45:05.128 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:45:05.128 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:45:05.128 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:45:05.128 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:45:05.128 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:45:05.128 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:45:05.128 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:45:05.128 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:45:05.128 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:04 smithi171 ceph-mon[31139]: Manager daemon y is now available 2024-04-26T04:45:05.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: Standby manager daemon x restarted 2024-04-26T04:45:05.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: Standby manager daemon x started 2024-04-26T04:45:05.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.? 172.21.15.171:0/4026315501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:45:05.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.? 172.21.15.171:0/4026315501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:45:05.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.? 172.21.15.171:0/4026315501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:45:05.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.? 172.21.15.171:0/4026315501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:45:05.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: Active manager daemon y restarted 2024-04-26T04:45:05.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: Activating manager daemon y 2024-04-26T04:45:05.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:45:05.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: mgrmap e35: y(active, starting, since 0.0068191s), standbys: x 2024-04-26T04:45:05.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:45:05.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:45:05.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:45:05.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.wyfzii"}]: dispatch 2024-04-26T04:45:05.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.kzjlqn"}]: dispatch 2024-04-26T04:45:05.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:45:05.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:45:05.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:45:05.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:45:05.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:45:05.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:45:05.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:45:05.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:45:05.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:45:05.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:45:05.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:45:05.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:45:05.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:45:05.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[98743]: Manager daemon y is now available 2024-04-26T04:45:05.346 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:45:04] ENGINE Bus STARTING 2024-04-26T04:45:05.346 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:05.008+0000 7f73bfaf1700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:05.346 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:05.008+0000 7f73bfaf1700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:05.347 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:05.008+0000 7f73bfaf1700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:05.347 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:05.008+0000 7f73bfaf1700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:05.347 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:05.008+0000 7f73bfaf1700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:05.347 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:05.008+0000 7f73bd2ec700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:05.347 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:05.008+0000 7f73bd2ec700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:05.347 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:05.008+0000 7f73bd2ec700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:05.347 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:05.008+0000 7f73bd2ec700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:05.347 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:05.008+0000 7f73bd2ec700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:05.347 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: CherryPy Checker: 2024-04-26T04:45:05.347 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: The Application mounted at '' has an empty config. 2024-04-26T04:45:05.347 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:05.347 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:45:05] ENGINE Serving on http://:::9283 2024-04-26T04:45:05.347 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:45:05] ENGINE Bus STARTED 2024-04-26T04:45:05.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: Standby manager daemon x restarted 2024-04-26T04:45:05.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: Standby manager daemon x started 2024-04-26T04:45:05.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/4026315501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:45:05.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/4026315501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:45:05.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/4026315501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:45:05.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.? 172.21.15.171:0/4026315501' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:45:05.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: Active manager daemon y restarted 2024-04-26T04:45:05.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: Activating manager daemon y 2024-04-26T04:45:05.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:45:05.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: mgrmap e35: y(active, starting, since 0.0068191s), standbys: x 2024-04-26T04:45:05.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:45:05.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:45:05.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:45:05.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.wyfzii"}]: dispatch 2024-04-26T04:45:05.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.kzjlqn"}]: dispatch 2024-04-26T04:45:05.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:45:05.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:45:05.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:45:05.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:45:05.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:45:05.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:45:05.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:45:05.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:45:05.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:45:05.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:45:05.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:45:05.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:45:05.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:45:05.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:04 smithi156 ceph-mon[39130]: Manager daemon y is now available 2024-04-26T04:45:05.499 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:05 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:45:05.900 INFO:teuthology.orchestra.run.smithi156.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:45:05.901 INFO:teuthology.orchestra.run.smithi156.stdout:alertmanager.a smithi156 *:9093,9094 running (84s) 43s ago 5m 25.7M - 0.25.0 c8568f914cd2 932345b7da32 2024-04-26T04:45:05.901 INFO:teuthology.orchestra.run.smithi156.stdout:grafana.a smithi171 *:3000 unknown 21s ago 5m - - 2024-04-26T04:45:05.901 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi156.wyfzii smithi156 running (4m) 43s ago 4m 15.1M - 17.2.7-902-gc61472c8 0de407c1b3af f42880bb83a3 2024-04-26T04:45:05.901 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi171.kzjlqn smithi171 running (4m) 21s ago 4m 13.7M - 17.2.7-902-gc61472c8 0de407c1b3af 31b518bc187c 2024-04-26T04:45:05.901 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.x smithi171 *:8443,9283,8765 running (26s) 21s ago 9m 67.5M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 9ec4d348d704 2024-04-26T04:45:05.901 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.y smithi156 *:8443,9283,8765 running (57s) 43s ago 11m 471M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 715bee4dd354 2024-04-26T04:45:05.901 INFO:teuthology.orchestra.run.smithi156.stdout:mon.a smithi156 running (11m) 43s ago 11m 51.2M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 3d899f93d0ef 2024-04-26T04:45:05.901 INFO:teuthology.orchestra.run.smithi156.stdout:mon.b smithi171 running (9m) 21s ago 9m 44.4M 2048M 17.2.7-902-gc61472c8 0de407c1b3af fb359e8b680c 2024-04-26T04:45:05.901 INFO:teuthology.orchestra.run.smithi156.stdout:mon.c smithi156 running (9m) 43s ago 9m 41.7M 2048M 17.2.7-902-gc61472c8 0de407c1b3af fc34a850791e 2024-04-26T04:45:05.901 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.a smithi156 *:9100 running (81s) 43s ago 4m 7931k - 1.5.0 0da6a335fe13 5af6998cc7e3 2024-04-26T04:45:05.901 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.b smithi171 *:9100 running (62s) 21s ago 4m 7759k - 1.5.0 0da6a335fe13 a2d37cfca8cf 2024-04-26T04:45:05.901 INFO:teuthology.orchestra.run.smithi156.stdout:osd.0 smithi156 running (9m) 43s ago 9m 46.6M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e1273966ada0 2024-04-26T04:45:05.901 INFO:teuthology.orchestra.run.smithi156.stdout:osd.1 smithi156 running (8m) 43s ago 8m 46.4M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 7abca656e994 2024-04-26T04:45:05.902 INFO:teuthology.orchestra.run.smithi156.stdout:osd.2 smithi156 running (8m) 43s ago 8m 52.3M 2480M 17.2.7-902-gc61472c8 0de407c1b3af c951e2171c5d 2024-04-26T04:45:05.902 INFO:teuthology.orchestra.run.smithi156.stdout:osd.3 smithi156 running (7m) 43s ago 7m 43.5M 2480M 17.2.7-902-gc61472c8 0de407c1b3af a2c01ca7770f 2024-04-26T04:45:05.902 INFO:teuthology.orchestra.run.smithi156.stdout:osd.4 smithi171 running (7m) 21s ago 7m 49.4M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 388a24a70001 2024-04-26T04:45:05.902 INFO:teuthology.orchestra.run.smithi156.stdout:osd.5 smithi171 running (7m) 21s ago 7m 52.5M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 703235a355b5 2024-04-26T04:45:05.902 INFO:teuthology.orchestra.run.smithi156.stdout:osd.6 smithi171 running (6m) 21s ago 6m 46.2M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 3dad877f8247 2024-04-26T04:45:05.902 INFO:teuthology.orchestra.run.smithi156.stdout:osd.7 smithi171 running (6m) 21s ago 6m 51.7M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e275b2da9b18 2024-04-26T04:45:05.902 INFO:teuthology.orchestra.run.smithi156.stdout:prometheus.a smithi171 *:9095 running (32s) 21s ago 5m 37.6M - 2.43.0 a07b618ecd1d 93d6b22f6fce 2024-04-26T04:45:06.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:05 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:05 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:05 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:45:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:05 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:45:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:05 smithi171 ceph-mon[31139]: [26/Apr/2024:04:45:05] ENGINE Bus STARTING 2024-04-26T04:45:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:05 smithi171 ceph-mon[31139]: [26/Apr/2024:04:45:05] ENGINE Serving on http://172.21.15.156:8765 2024-04-26T04:45:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:05 smithi171 ceph-mon[31139]: [26/Apr/2024:04:45:05] ENGINE Serving on https://172.21.15.156:7150 2024-04-26T04:45:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:05 smithi171 ceph-mon[31139]: [26/Apr/2024:04:45:05] ENGINE Bus STARTED 2024-04-26T04:45:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:05 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:05 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:05 smithi171 ceph-mon[31139]: mgrmap e36: y(active, since 1.0106s), standbys: x 2024-04-26T04:45:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:05 smithi171 ceph-mon[31139]: from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:45:06.251 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:05 smithi171 ceph-mon[31139]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:06.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:45:05.890+0000 7f741c426700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-04-26T04:45:06.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:45:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:45:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[39130]: [26/Apr/2024:04:45:05] ENGINE Bus STARTING 2024-04-26T04:45:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[39130]: [26/Apr/2024:04:45:05] ENGINE Serving on http://172.21.15.156:8765 2024-04-26T04:45:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[39130]: [26/Apr/2024:04:45:05] ENGINE Serving on https://172.21.15.156:7150 2024-04-26T04:45:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[39130]: [26/Apr/2024:04:45:05] ENGINE Bus STARTED 2024-04-26T04:45:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[39130]: mgrmap e36: y(active, since 1.0106s), standbys: x 2024-04-26T04:45:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[39130]: from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:45:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[39130]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:06.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:06.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:06.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:45:06.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:45:06.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[98743]: [26/Apr/2024:04:45:05] ENGINE Bus STARTING 2024-04-26T04:45:06.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[98743]: [26/Apr/2024:04:45:05] ENGINE Serving on http://172.21.15.156:8765 2024-04-26T04:45:06.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[98743]: [26/Apr/2024:04:45:05] ENGINE Serving on https://172.21.15.156:7150 2024-04-26T04:45:06.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[98743]: [26/Apr/2024:04:45:05] ENGINE Bus STARTED 2024-04-26T04:45:06.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:06.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:06.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[98743]: mgrmap e36: y(active, since 1.0106s), standbys: x 2024-04-26T04:45:06.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[98743]: from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:45:06.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:05 smithi156 ceph-mon[98743]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:06.773 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T04:45:07.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:06 smithi156 ceph-mon[98743]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:45:07.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:06 smithi156 ceph-mon[98743]: Cluster is now healthy 2024-04-26T04:45:07.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:06 smithi156 ceph-mon[98743]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:07.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:06 smithi156 ceph-mon[39130]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:45:07.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:06 smithi156 ceph-mon[39130]: Cluster is now healthy 2024-04-26T04:45:07.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:06 smithi156 ceph-mon[39130]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:07.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:06 smithi171 ceph-mon[31139]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:45:07.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:06 smithi171 ceph-mon[31139]: Cluster is now healthy 2024-04-26T04:45:07.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:06 smithi171 ceph-mon[31139]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:07.281 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:08.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:07 smithi156 ceph-mon[39130]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-26T04:45:08.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:07 smithi156 ceph-mon[98743]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-26T04:45:08.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:07 smithi171 ceph-mon[31139]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-26T04:45:09.088 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:08 smithi156 ceph-mon[98743]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:09.088 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:08 smithi156 ceph-mon[39130]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:09.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:08 smithi171 ceph-mon[31139]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:09.840 INFO:teuthology.orchestra.run.smithi156.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:45:10.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:09 smithi156 ceph-mon[98743]: mgrmap e38: y(active, since 4s), standbys: x 2024-04-26T04:45:10.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:09 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:10.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:09 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:10.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:09 smithi156 ceph-mon[98743]: from='client.25304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:45:10.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:09 smithi156 ceph-mon[98743]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:45:10.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:09 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:10.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:09 smithi156 ceph-mon[39130]: mgrmap e38: y(active, since 4s), standbys: x 2024-04-26T04:45:10.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:09 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:10.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:09 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:10.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:09 smithi156 ceph-mon[39130]: from='client.25304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:45:10.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:09 smithi156 ceph-mon[39130]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:45:10.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:09 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:10.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:09 smithi171 ceph-mon[31139]: mgrmap e38: y(active, since 4s), standbys: x 2024-04-26T04:45:10.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:09 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:10.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:09 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:10.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:09 smithi171 ceph-mon[31139]: from='client.25304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:45:10.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:09 smithi171 ceph-mon[31139]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:45:10.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:09 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:11.154 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 60' 2024-04-26T04:45:11.919 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:11.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:11 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:11.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:11 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:11.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:11 smithi171 ceph-mon[31139]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:12.073 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:11 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:12.073 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:11 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:12.073 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:11 smithi156 ceph-mon[98743]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:12.073 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:11 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:12.073 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:11 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:12.073 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:11 smithi156 ceph-mon[39130]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:13.492 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:13 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:13.492 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:13 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:13.492 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:13 smithi156 ceph-mon[98743]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:45:13.493 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:13 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:13.493 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:13 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:13.493 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:13 smithi156 ceph-mon[39130]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:45:13.493 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:13 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:13.493 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:13 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:13.493 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:13 smithi171 ceph-mon[31139]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:45:14.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:14 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:14.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:14 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:14.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:14 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:14.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:14 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:14.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:14 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:14.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:14 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:14 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:14 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:14 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:14 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:14 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:14 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:14.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:14 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:14.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:14 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:14.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:14 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:14.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:14 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:14.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:14 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:14.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:14 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:15.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:45:14] "GET /metrics HTTP/1.1" 200 31151 "" "Prometheus/2.43.0" 2024-04-26T04:45:15.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:15 smithi171 ceph-mon[31139]: Updating smithi156:/etc/ceph/ceph.conf 2024-04-26T04:45:15.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:15 smithi171 ceph-mon[31139]: Updating smithi171:/etc/ceph/ceph.conf 2024-04-26T04:45:15.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:15 smithi171 ceph-mon[31139]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:45:15.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:15 smithi171 ceph-mon[31139]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:45:15.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:15 smithi171 ceph-mon[31139]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:45:15.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:15 smithi171 ceph-mon[31139]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:45:15.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:15 smithi171 ceph-mon[31139]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:45:15.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:15 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:15.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:15 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:15.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:15 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:15.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:15 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:15.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:15 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:15.522 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:15 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 6. 2024-04-26T04:45:15.522 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:15 smithi171 systemd[1]: Stopped Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:45:15.522 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:15 smithi171 systemd[1]: Starting Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:45:15.522 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:15 smithi171 podman[88236]: 2024-04-26 04:45:15.381624026 +0000 UTC m=+0.017275465 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:45:15.522 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:15 smithi171 podman[88236]: 2024-04-26 04:45:15.52047362 +0000 UTC m=+0.156125101 container create 97dd47edd7ad17201ff19cbf70d3733e0b3ffa4113c231d12faef565ac9c3a44 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , release=239.1651231664, io.openshift.expose-services=, version=8.5, description=Ceph Grafana Container, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-26T04:45:15.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[39130]: Updating smithi156:/etc/ceph/ceph.conf 2024-04-26T04:45:15.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[39130]: Updating smithi171:/etc/ceph/ceph.conf 2024-04-26T04:45:15.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[39130]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:45:15.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[39130]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:45:15.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[39130]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:45:15.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[39130]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:45:15.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[39130]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:45:15.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:15.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:15.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:15.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:15.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:15.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[98743]: Updating smithi156:/etc/ceph/ceph.conf 2024-04-26T04:45:15.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[98743]: Updating smithi171:/etc/ceph/ceph.conf 2024-04-26T04:45:15.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[98743]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:45:15.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[98743]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:45:15.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[98743]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:45:15.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[98743]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:45:15.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[98743]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:45:15.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:15.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:15.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:15.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:15.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:15 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:15.831 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:15 smithi171 podman[88236]: 2024-04-26 04:45:15.756106401 +0000 UTC m=+0.391757840 container init 97dd47edd7ad17201ff19cbf70d3733e0b3ffa4113c231d12faef565ac9c3a44 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, version=8.5, vcs-type=git, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, maintainer=Paul Cuzner , io.openshift.expose-services=, description=Ceph Grafana Container, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, vendor=Red Hat, Inc.) 2024-04-26T04:45:15.831 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:15 smithi171 podman[88236]: 2024-04-26 04:45:15.761858834 +0000 UTC m=+0.397510274 container start 97dd47edd7ad17201ff19cbf70d3733e0b3ffa4113c231d12faef565ac9c3a44 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, description=Ceph Grafana Container, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, version=8.5, distribution-scope=public, build-date=2022-04-29T11:48:11.055897) 2024-04-26T04:45:16.235 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:15 smithi171 bash[88236]: 97dd47edd7ad17201ff19cbf70d3733e0b3ffa4113c231d12faef565ac9c3a44 2024-04-26T04:45:16.235 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:15 smithi171 systemd[1]: Started Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:45:16.235 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:15 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[88283]: logger=settings t=2024-04-26T04:45:15.91516568Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:45:16.235 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:15 smithi171 podman[88320]: 2024-04-26 04:45:15.941167429 +0000 UTC m=+0.018745620 container died 97dd47edd7ad17201ff19cbf70d3733e0b3ffa4113c231d12faef565ac9c3a44 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, io.openshift.expose-services=, description=Ceph Grafana Container, io.buildah.version=1.32.0, name=ubi8, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, distribution-scope=public, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-26T04:45:16.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:16 smithi171 ceph-mon[31139]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:45:16.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:16 smithi171 ceph-mon[31139]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:45:16.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:16 smithi171 ceph-mon[31139]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:45:16.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:16 smithi171 ceph-mon[31139]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:45:16.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:16 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:16.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:16 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:16.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:16 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:16.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:16 smithi171 ceph-mon[31139]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T04:45:16.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:16 smithi171 ceph-mon[31139]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T04:45:16.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:16 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:16.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[39130]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:45:16.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[39130]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:45:16.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[39130]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:45:16.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[39130]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:45:16.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:16.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:16.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:16.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[39130]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T04:45:16.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[39130]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T04:45:16.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:16.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[98743]: Updating smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:45:16.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[98743]: Updating smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:45:16.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[98743]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:45:16.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[98743]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:45:16.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:16.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:16.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:16.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[98743]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-26T04:45:16.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[98743]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T04:45:16.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:16 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:45:16.999 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:16 smithi171 podman[88320]: 2024-04-26 04:45:16.534635717 +0000 UTC m=+0.612213903 container remove 97dd47edd7ad17201ff19cbf70d3733e0b3ffa4113c231d12faef565ac9c3a44 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, architecture=x86_64, io.openshift.expose-services=, release=239.1651231664, distribution-scope=public, vcs-type=git, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, maintainer=Paul Cuzner , name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-26T04:45:17.000 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:16 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:45:17.000 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:16 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:45:18.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:18 smithi171 ceph-mon[31139]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T04:45:18.750 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:45:18 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:45:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:45:18.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:18 smithi156 ceph-mon[39130]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T04:45:18.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:18 smithi156 ceph-mon[98743]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T04:45:20.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:20 smithi171 ceph-mon[31139]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:20.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:20 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:20.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:20 smithi156 ceph-mon[39130]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:20.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:20 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:20.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:20 smithi156 ceph-mon[98743]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:20.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:22.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:22 smithi171 ceph-mon[31139]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:22.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:22 smithi156 ceph-mon[98743]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:22.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:22 smithi156 ceph-mon[39130]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:24.704 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:24 smithi156 ceph-mon[98743]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:24.704 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:24 smithi156 ceph-mon[39130]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:24.748 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:24 smithi171 ceph-mon[31139]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:25.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:45:24] "GET /metrics HTTP/1.1" 200 33788 "" "Prometheus/2.43.0" 2024-04-26T04:45:26.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:26 smithi171 ceph-mon[31139]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:26.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:26 smithi156 ceph-mon[98743]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:26.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:26 smithi156 ceph-mon[39130]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:27.356 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:27 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 7. 2024-04-26T04:45:27.356 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:27 smithi171 systemd[1]: Stopped Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:45:27.357 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:27 smithi171 systemd[1]: Starting Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:45:27.357 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:27 smithi171 podman[88432]: 2024-04-26 04:45:27.135140995 +0000 UTC m=+0.017550073 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:45:27.357 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:27 smithi171 podman[88432]: 2024-04-26 04:45:27.243841246 +0000 UTC m=+0.126250306 container create 76466cb7f733262c4cd868e5535738e9df844b2a3f913110a291d5c39fe2ea99 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, io.openshift.expose-services=, description=Ceph Grafana Container, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , distribution-scope=public, vcs-type=git, build-date=2022-04-29T11:48:11.055897) 2024-04-26T04:45:27.749 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:27 smithi171 podman[88432]: 2024-04-26 04:45:27.462796537 +0000 UTC m=+0.345205601 container init 76466cb7f733262c4cd868e5535738e9df844b2a3f913110a291d5c39fe2ea99 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.openshift.tags=base rhel8, architecture=x86_64, release=239.1651231664, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, version=8.5, vcs-type=git, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , vendor=Red Hat, Inc., io.openshift.expose-services=) 2024-04-26T04:45:27.750 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:27 smithi171 podman[88432]: 2024-04-26 04:45:27.468278114 +0000 UTC m=+0.350687175 container start 76466cb7f733262c4cd868e5535738e9df844b2a3f913110a291d5c39fe2ea99 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, architecture=x86_64, release=239.1651231664, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, vendor=Red Hat, Inc., maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, description=Ceph Grafana Container, vcs-type=git, name=ubi8, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:45:27.750 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:27 smithi171 bash[88432]: 76466cb7f733262c4cd868e5535738e9df844b2a3f913110a291d5c39fe2ea99 2024-04-26T04:45:27.750 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:27 smithi171 systemd[1]: Started Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:45:27.750 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:27 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a[88447]: logger=settings t=2024-04-26T04:45:27.620089807Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:45:27.750 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:27 smithi171 podman[88484]: 2024-04-26 04:45:27.646187962 +0000 UTC m=+0.018898865 container died 76466cb7f733262c4cd868e5535738e9df844b2a3f913110a291d5c39fe2ea99 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, name=ubi8, distribution-scope=public, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, io.openshift.expose-services=, vendor=Red Hat, Inc., release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-26T04:45:28.659 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:28 smithi171 ceph-mon[31139]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:28.659 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:28 smithi171 podman[88484]: 2024-04-26 04:45:28.265902427 +0000 UTC m=+0.638613327 container remove 76466cb7f733262c4cd868e5535738e9df844b2a3f913110a291d5c39fe2ea99 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., io.buildah.version=1.32.0, description=Ceph Grafana Container, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, build-date=2022-04-29T11:48:11.055897, name=ubi8, vcs-type=git, io.openshift.tags=base rhel8, io.openshift.expose-services=) 2024-04-26T04:45:28.659 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:28 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:45:28.660 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:28 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:45:28.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:28 smithi156 ceph-mon[98743]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:28.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:28 smithi156 ceph-mon[39130]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:28.999 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:45:28 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:45:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:45:30.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:30 smithi171 ceph-mon[31139]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:30.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:30 smithi156 ceph-mon[39130]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:30.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:30 smithi156 ceph-mon[98743]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:32.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:32 smithi171 ceph-mon[31139]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:32 smithi156 ceph-mon[98743]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:32.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:32 smithi156 ceph-mon[39130]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:34.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:34 smithi171 ceph-mon[31139]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:34.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:34 smithi156 ceph-mon[98743]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:34.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:34 smithi156 ceph-mon[39130]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:34.844 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:45:34] "GET /metrics HTTP/1.1" 200 33788 "" "Prometheus/2.43.0" 2024-04-26T04:45:35.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:35 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:35.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:35.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:35 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:36.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:36 smithi171 ceph-mon[31139]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:36.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:36 smithi156 ceph-mon[98743]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:36.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:36 smithi156 ceph-mon[39130]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:38.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:38 smithi171 ceph-mon[31139]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:38.750 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:45:38 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:45:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:45:38.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:38 smithi156 ceph-mon[98743]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:38.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:38 smithi156 ceph-mon[39130]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:39.249 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:38 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 8. 2024-04-26T04:45:39.249 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:38 smithi171 systemd[1]: Stopped Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:45:39.250 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:38 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Start request repeated too quickly. 2024-04-26T04:45:39.250 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:38 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:45:39.250 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Apr 26 04:45:38 smithi171 systemd[1]: Failed to start Ceph grafana.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:45:40.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:40 smithi171 ceph-mon[31139]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:40.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:40 smithi156 ceph-mon[98743]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:40.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:40 smithi156 ceph-mon[39130]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:42.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:42 smithi171 ceph-mon[31139]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:42.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:42 smithi156 ceph-mon[98743]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:42.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:42 smithi156 ceph-mon[39130]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:44.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:44 smithi171 ceph-mon[31139]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:44.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:44 smithi156 ceph-mon[98743]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:44.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:45:44] "GET /metrics HTTP/1.1" 200 33789 "" "Prometheus/2.43.0" 2024-04-26T04:45:44.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:44 smithi156 ceph-mon[39130]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:46.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:46 smithi171 ceph-mon[31139]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:46.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:46 smithi156 ceph-mon[98743]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:46.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:46 smithi156 ceph-mon[39130]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:48.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:48 smithi171 ceph-mon[31139]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:48.750 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:45:48 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:45:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:45:48.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:48 smithi156 ceph-mon[98743]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:48.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:48 smithi156 ceph-mon[39130]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:51.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:50 smithi171 ceph-mon[31139]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:51.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:50 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:51.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:50 smithi156 ceph-mon[39130]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:51.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:50 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:51.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:50 smithi156 ceph-mon[98743]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:51.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:50 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:53.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:52 smithi171 ceph-mon[31139]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:53.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:52 smithi156 ceph-mon[98743]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:53.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:52 smithi156 ceph-mon[39130]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:55.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:54 smithi156 ceph-mon[39130]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:55.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:54 smithi156 ceph-mon[98743]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:55.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:45:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:45:54] "GET /metrics HTTP/1.1" 200 33790 "" "Prometheus/2.43.0" 2024-04-26T04:45:55.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:54 smithi171 ceph-mon[31139]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:57.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:56 smithi171 ceph-mon[31139]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:57.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:56 smithi156 ceph-mon[98743]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:57.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:56 smithi156 ceph-mon[39130]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:58.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:45:58 smithi171 ceph-mon[31139]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:59.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:45:58 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:45:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:45:59.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:45:58 smithi156 ceph-mon[98743]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:59.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:45:58 smithi156 ceph-mon[39130]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:01.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:00 smithi171 ceph-mon[31139]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:01.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:00 smithi156 ceph-mon[39130]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:01.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:00 smithi156 ceph-mon[98743]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:03.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:02 smithi171 ceph-mon[31139]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:03.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:02 smithi156 ceph-mon[98743]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:03.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:02 smithi156 ceph-mon[39130]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:05.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:04 smithi156 ceph-mon[39130]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:05.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:04 smithi156 ceph-mon[98743]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:05.094 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:46:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:46:04] "GET /metrics HTTP/1.1" 200 33790 "" "Prometheus/2.43.0" 2024-04-26T04:46:05.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:04 smithi171 ceph-mon[31139]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:06.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:05 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:06.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:05 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:06.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:07.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:06 smithi171 ceph-mon[31139]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:07.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:06 smithi156 ceph-mon[98743]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:07.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:06 smithi156 ceph-mon[39130]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:08.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:08 smithi171 ceph-mon[31139]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:09.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:46:08 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:46:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:46:09.146 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:08 smithi156 ceph-mon[39130]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:09.146 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:08 smithi156 ceph-mon[98743]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:11.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:10 smithi171 ceph-mon[31139]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:11.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:10 smithi156 ceph-mon[39130]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:11.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:10 smithi156 ceph-mon[98743]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:13.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:12 smithi171 ceph-mon[31139]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:13.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:12 smithi156 ceph-mon[98743]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:13.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:12 smithi156 ceph-mon[39130]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:14.376 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade resume' 2024-04-26T04:46:14.639 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:14.956 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:14 smithi156 ceph-mon[98743]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:14.956 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:46:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:46:14] "GET /metrics HTTP/1.1" 200 33789 "" "Prometheus/2.43.0" 2024-04-26T04:46:14.956 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:14 smithi156 ceph-mon[39130]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:15.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:14 smithi171 ceph-mon[31139]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:16.234 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:15 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:16.235 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:15 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:16.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:15 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:16.692 INFO:teuthology.orchestra.run.smithi156.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:46:17.063 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:16 smithi156 ceph-mon[98743]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:17.063 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:16 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:17.063 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:16 smithi156 ceph-mon[39130]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:17.063 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:16 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:17.168 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:16 smithi171 ceph-mon[31139]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:17.168 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:16 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:17.362 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for majority of mons to upgrade' 2024-04-26T04:46:17.813 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:18.142 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:17 smithi156 ceph-mon[98743]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:18.142 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:17 smithi156 ceph-mon[98743]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:46:18.142 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:17 smithi156 ceph-mon[39130]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:18.142 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:17 smithi156 ceph-mon[39130]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:46:18.234 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:17 smithi171 ceph-mon[31139]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:18.234 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:17 smithi171 ceph-mon[31139]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:46:18.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:18 smithi171 ceph-mon[31139]: pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:19.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:46:18 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:46:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:46:19.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:18 smithi156 ceph-mon[98743]: pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:19.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:18 smithi156 ceph-mon[39130]: pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:20.097 INFO:teuthology.orchestra.run.smithi156.stdout:wait for majority of mons to upgrade 2024-04-26T04:46:20.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:20 smithi171 ceph-mon[31139]: pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:20.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:20 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:20.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:20 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:20.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:20 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:20.802 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:20 smithi156 ceph-mon[98743]: pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:20.802 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:20.802 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:20.802 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:20.803 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:20 smithi156 ceph-mon[39130]: pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:20.803 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:20 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:20.803 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:20 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:20.803 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:20 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:20.827 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-04-26T04:46:21.679 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:22.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:22 smithi171 ceph-mon[31139]: pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:22.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:22 smithi156 ceph-mon[98743]: pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:22.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:22 smithi156 ceph-mon[39130]: pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:24.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:24 smithi171 ceph-mon[31139]: pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:24.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:24 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1642890007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:24.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:24 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:24.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:24 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:24.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:46:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:46:24] "GET /metrics HTTP/1.1" 200 33786 "" "Prometheus/2.43.0" 2024-04-26T04:46:24.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:24 smithi156 ceph-mon[39130]: pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:24.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:24 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/1642890007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:24.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:24 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:24.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:24 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:24.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:24 smithi156 ceph-mon[98743]: pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:24.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:24 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1642890007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:24.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:24 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:24.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:24 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:25.583 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:25.583 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:25.583 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[39130]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:25.583 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[39130]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:25.583 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:25.583 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:25.583 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:46:25.583 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:46:25.583 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T04:46:25.583 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[39130]: Upgrade: It appears safe to stop mon.c 2024-04-26T04:46:25.584 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:25.584 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:25.584 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[98743]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:25.584 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[98743]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:25.584 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:25.584 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:25.584 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:46:25.584 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:46:25.584 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T04:46:25.584 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:25 smithi156 ceph-mon[98743]: Upgrade: It appears safe to stop mon.c 2024-04-26T04:46:25.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:25 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:25.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:25 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:25.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:25 smithi171 ceph-mon[31139]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:25.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:25 smithi171 ceph-mon[31139]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:25.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:25 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:25.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:25 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:25.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:25 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:46:25.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:25 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:46:25.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:25 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T04:46:25.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:25 smithi171 ceph-mon[31139]: Upgrade: It appears safe to stop mon.c 2024-04-26T04:46:26.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:26 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/179609552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:26.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:26 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:26.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:26 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:46:26.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:26 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:46:26.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:26 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:26.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:26 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/179609552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:26.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:26 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:26.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:26 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:46:26.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:26 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:46:26.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:26 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:26.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:26 smithi156 ceph-mon[39130]: from='client.? 172.21.15.156:0/179609552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:26.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:26 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:26.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:26 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:46:26.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:26 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:46:26.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:26 smithi156 ceph-mon[39130]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:27.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:27 smithi156 ceph-mon[98743]: Upgrade: Updating mon.c 2024-04-26T04:46:27.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:27 smithi156 ceph-mon[98743]: Deploying daemon mon.c on smithi156 2024-04-26T04:46:27.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:27 smithi156 ceph-mon[98743]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:27.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:27 smithi156 ceph-mon[39130]: Upgrade: Updating mon.c 2024-04-26T04:46:27.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:27 smithi156 ceph-mon[39130]: Deploying daemon mon.c on smithi156 2024-04-26T04:46:27.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:27 smithi156 ceph-mon[39130]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:27.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:27 smithi171 ceph-mon[31139]: Upgrade: Updating mon.c 2024-04-26T04:46:27.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:27 smithi171 ceph-mon[31139]: Deploying daemon mon.c on smithi156 2024-04-26T04:46:27.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:27 smithi171 ceph-mon[31139]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:28.570 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:28 smithi156 systemd[1]: Stopping Ceph mon.c for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:46:28.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:28 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c[39107]: 2024-04-26T04:46:28.567+0000 7f9c688ee700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:46:28.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:28 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c[39107]: 2024-04-26T04:46:28.567+0000 7f9c688ee700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-04-26T04:46:28.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:28 smithi156 podman[103070]: 2024-04-26 04:46:28.730915887 +0000 UTC m=+0.279401859 container died fc34a850791e9377c80d935c903d844b5d0ee2fe0ef2d6d2cab230f7534c9c0e (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-26T04:46:29.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:46:28 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:46:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:46:29.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:29 smithi156 podman[103070]: 2024-04-26 04:46:29.207475077 +0000 UTC m=+0.755961050 container cleanup fc34a850791e9377c80d935c903d844b5d0ee2fe0ef2d6d2cab230f7534c9c0e (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c) 2024-04-26T04:46:29.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:29 smithi156 bash[103070]: ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c 2024-04-26T04:46:29.823 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:29 smithi156 podman[103109]: 2024-04-26 04:46:29.400655335 +0000 UTC m=+0.670783855 container remove fc34a850791e9377c80d935c903d844b5d0ee2fe0ef2d6d2cab230f7534c9c0e (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True) 2024-04-26T04:46:29.823 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:29 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.c.service: Deactivated successfully. 2024-04-26T04:46:29.823 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:29 smithi156 systemd[1]: Stopped Ceph mon.c for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:46:29.823 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:29 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.c.service: Consumed 7.381s CPU time. 2024-04-26T04:46:30.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:29 smithi156 systemd[1]: Starting Ceph mon.c for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:46:30.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 podman[103251]: 2024-04-26 04:46:29.941043161 +0000 UTC m=+0.017843926 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:46:30.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 podman[103251]: 2024-04-26 04:46:30.051774536 +0000 UTC m=+0.128575287 container create 8cac6c96e0db9048064ab747ea8d4ffcc421a7f45968df33dc89a98301a05e07 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424) 2024-04-26T04:46:30.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 podman[103251]: 2024-04-26 04:46:30.262852812 +0000 UTC m=+0.339653565 container init 8cac6c96e0db9048064ab747ea8d4ffcc421a7f45968df33dc89a98301a05e07 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:46:30.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 podman[103251]: 2024-04-26 04:46:30.26838964 +0000 UTC m=+0.345190392 container start 8cac6c96e0db9048064ab747ea8d4ffcc421a7f45968df33dc89a98301a05e07 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240424) 2024-04-26T04:46:30.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:46:30.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 8 2024-04-26T04:46:30.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: pidfile_write: ignore empty --pid-file 2024-04-26T04:46:30.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: load: jerasure load: lrc 2024-04-26T04:46:30.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: RocksDB version: 7.9.2 2024-04-26T04:46:30.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Git sha 0 2024-04-26T04:46:30.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-26T04:46:30.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: DB SUMMARY 2024-04-26T04:46:30.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: DB Session ID: M4QMGH75NZ43KCEJ6HQ2 2024-04-26T04:46:30.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:46:30.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:46:30.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1084 Bytes 2024-04-26T04:46:30.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-04-26T04:46:30.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 1475174 ; 2024-04-26T04:46:30.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:46:30.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:46:30.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:46:30.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T04:46:30.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:46:30.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T04:46:30.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.env: 0x56063c307c60 2024-04-26T04:46:30.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T04:46:30.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.info_log: 0x56063e199aa0 2024-04-26T04:46:30.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:46:30.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.statistics: (nil) 2024-04-26T04:46:30.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.use_fsync: 0 2024-04-26T04:46:30.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:46:30.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:46:30.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:46:30.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:46:30.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:46:30.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:46:30.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:46:30.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:46:30.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:46:30.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:46:30.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:46:30.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.db_log_dir: 2024-04-26T04:46:30.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.wal_dir: 2024-04-26T04:46:30.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:46:30.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:46:30.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:46:30.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:46:30.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:46:30.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:46:30.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:46:30.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:46:30.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.write_buffer_manager: 0x56063e16c820 2024-04-26T04:46:30.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:46:30.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:46:30.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:46:30.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:46:30.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:46:30.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:46:30.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:46:30.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:46:30.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.unordered_write: 0 2024-04-26T04:46:30.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:46:30.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:46:30.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:46:30.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:46:30.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.row_cache: None 2024-04-26T04:46:30.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.wal_filter: None 2024-04-26T04:46:30.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:46:30.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:46:30.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:46:30.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:46:30.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.wal_compression: 0 2024-04-26T04:46:30.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:46:30.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:46:30.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:46:30.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:46:30.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:46:30.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:46:30.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:46:30.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:46:30.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:46:30.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:46:30.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:46:30.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T04:46:30.602 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:46:30.602 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:46:30.602 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:46:30.602 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:46:30.602 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:46:30.602 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:46:30.602 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:46:30.602 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:46:30.602 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:46:30.602 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:46:30.602 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:46:30.602 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_open_files: -1 2024-04-26T04:46:30.602 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:46:30.603 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:46:30.603 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:46:30.603 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:46:30.603 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:46:30.603 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Compression algorithms supported: 2024-04-26T04:46:30.603 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:46:30.603 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: kZSTD supported: 0 2024-04-26T04:46:30.603 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:46:30.603 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:46:30.603 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:46:30.603 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:46:30.603 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:46:30.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:46:30.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:46:30.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T04:46:30.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-04-26T04:46:30.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T04:46:30.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:46:30.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.merge_operator: 2024-04-26T04:46:30.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compaction_filter: None 2024-04-26T04:46:30.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:46:30.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:46:30.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:46:30.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:46:30.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56063e199ba0) 2024-04-26T04:46:30.604 INFO:journalctl@ceph.mon.c.smithi156.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:46:30.605 INFO:journalctl@ceph.mon.c.smithi156.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:46:30.605 INFO:journalctl@ceph.mon.c.smithi156.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:46:30.605 INFO:journalctl@ceph.mon.c.smithi156.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:46:30.605 INFO:journalctl@ceph.mon.c.smithi156.stdout: index_type: 0 2024-04-26T04:46:30.605 INFO:journalctl@ceph.mon.c.smithi156.stdout: data_block_index_type: 0 2024-04-26T04:46:30.605 INFO:journalctl@ceph.mon.c.smithi156.stdout: index_shortening: 1 2024-04-26T04:46:30.605 INFO:journalctl@ceph.mon.c.smithi156.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:46:30.605 INFO:journalctl@ceph.mon.c.smithi156.stdout: checksum: 4 2024-04-26T04:46:30.605 INFO:journalctl@ceph.mon.c.smithi156.stdout: no_block_cache: 0 2024-04-26T04:46:30.605 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_cache: 0x56063e19d090 2024-04-26T04:46:30.605 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:46:30.605 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_cache_options: 2024-04-26T04:46:30.605 INFO:journalctl@ceph.mon.c.smithi156.stdout: capacity : 536870912 2024-04-26T04:46:30.605 INFO:journalctl@ceph.mon.c.smithi156.stdout: num_shard_bits : 4 2024-04-26T04:46:30.605 INFO:journalctl@ceph.mon.c.smithi156.stdout: strict_capacity_limit : 0 2024-04-26T04:46:30.606 INFO:journalctl@ceph.mon.c.smithi156.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:46:30.606 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_cache_compressed: (nil) 2024-04-26T04:46:30.606 INFO:journalctl@ceph.mon.c.smithi156.stdout: persistent_cache: (nil) 2024-04-26T04:46:30.606 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_size: 4096 2024-04-26T04:46:30.606 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_size_deviation: 10 2024-04-26T04:46:30.606 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_restart_interval: 16 2024-04-26T04:46:30.606 INFO:journalctl@ceph.mon.c.smithi156.stdout: index_block_restart_interval: 1 2024-04-26T04:46:30.606 INFO:journalctl@ceph.mon.c.smithi156.stdout: metadata_block_size: 4096 2024-04-26T04:46:30.606 INFO:journalctl@ceph.mon.c.smithi156.stdout: partition_filters: 0 2024-04-26T04:46:30.606 INFO:journalctl@ceph.mon.c.smithi156.stdout: use_delta_encoding: 1 2024-04-26T04:46:30.606 INFO:journalctl@ceph.mon.c.smithi156.stdout: filter_policy: bloomfilter 2024-04-26T04:46:30.606 INFO:journalctl@ceph.mon.c.smithi156.stdout: whole_key_filtering: 1 2024-04-26T04:46:30.607 INFO:journalctl@ceph.mon.c.smithi156.stdout: verify_compression: 0 2024-04-26T04:46:30.607 INFO:journalctl@ceph.mon.c.smithi156.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:46:30.607 INFO:journalctl@ceph.mon.c.smithi156.stdout: format_version: 5 2024-04-26T04:46:30.607 INFO:journalctl@ceph.mon.c.smithi156.stdout: enable_index_compression: 1 2024-04-26T04:46:30.607 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_align: 0 2024-04-26T04:46:30.607 INFO:journalctl@ceph.mon.c.smithi156.stdout: max_auto_readahead_size: 262144 2024-04-26T04:46:30.607 INFO:journalctl@ceph.mon.c.smithi156.stdout: prepopulate_block_cache: 0 2024-04-26T04:46:30.607 INFO:journalctl@ceph.mon.c.smithi156.stdout: initial_auto_readahead_size: 8192 2024-04-26T04:46:30.607 INFO:journalctl@ceph.mon.c.smithi156.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T04:46:30.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:46:30.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:46:30.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compression: NoCompression 2024-04-26T04:46:30.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:46:30.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:46:30.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:46:30.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.num_levels: 7 2024-04-26T04:46:30.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:46:30.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:46:30.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:46:30.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:46:30.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:46:30.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:46:30.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:46:30.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:46:30.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:46:30.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:46:30.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:46:30.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:46:30.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:46:30.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:46:30.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:46:30.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:46:30.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:46:30.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:46:30.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:46:30.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:46:30.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:46:30.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:46:30.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:46:30.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:46:30.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:46:30.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:46:30.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:46:30.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:46:30.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:46:30.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:46:30.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:46:30.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:46:30.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:46:30.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:46:30.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:46:30.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:46:30.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:46:30.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:46:30.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T04:46:30.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T04:46:30.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:46:30.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:46:30.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:46:30.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:46:30.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:46:30.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:46:30.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:46:30.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:46:30.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:46:30.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:46:30.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:46:30.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:46:30.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:46:30.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:46:30.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:46:30.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:46:30.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:46:30.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:46:30.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:46:30.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:46:30.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:46:30.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:46:30.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:46:30.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:46:30.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:46:30.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.ttl: 2592000 2024-04-26T04:46:30.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:46:30.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T04:46:30.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T04:46:30.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:46:30.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:46:30.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:46:30.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:46:30.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:46:30.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:46:30.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T04:46:30.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T04:46:30.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T04:46:30.615 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T04:46:30.615 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-04-26T04:46:30.615 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:46:30.615 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12925, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:46:30.615 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-26T04:46:30.615 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: b64b9110-2f6d-4d32-a695-922085150e22 2024-04-26T04:46:30.615 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106790297775, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-26T04:46:30.615 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-26T04:46:30.615 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106790302430, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 894287, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12930, "largest_seqno": 13870, "table_properties": {"data_size": 889673, "index_size": 2704, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 901, "raw_key_size": 7592, "raw_average_key_size": 21, "raw_value_size": 881846, "raw_average_value_size": 2541, "num_data_blocks": 123, "num_entries": 347, "num_filter_entries": 347, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714106790, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "b64b9110-2f6d-4d32-a695-922085150e22", "db_session_id": "M4QMGH75NZ43KCEJ6HQ2", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-26T04:46:30.615 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106790302542, "job": 1, "event": "recovery_finished"} 2024-04-26T04:46:30.615 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-26T04:46:30.616 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:46:30.616 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:46:30.616 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56063e28e000 2024-04-26T04:46:30.616 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: DB pointer 0x56063e278000 2024-04-26T04:46:30.616 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T04:46:30.616 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T04:46:30.616 INFO:journalctl@ceph.mon.c.smithi156.stdout: ** DB Stats ** 2024-04-26T04:46:30.616 INFO:journalctl@ceph.mon.c.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:46:30.616 INFO:journalctl@ceph.mon.c.smithi156.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:46:30.616 INFO:journalctl@ceph.mon.c.smithi156.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:46:30.616 INFO:journalctl@ceph.mon.c.smithi156.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:46:30.616 INFO:journalctl@ceph.mon.c.smithi156.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:46:30.616 INFO:journalctl@ceph.mon.c.smithi156.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:46:30.616 INFO:journalctl@ceph.mon.c.smithi156.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:46:30.616 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-04-26T04:46:30.617 INFO:journalctl@ceph.mon.c.smithi156.stdout: ** Compaction Stats [default] ** 2024-04-26T04:46:30.617 INFO:journalctl@ceph.mon.c.smithi156.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:46:30.617 INFO:journalctl@ceph.mon.c.smithi156.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T04:46:30.617 INFO:journalctl@ceph.mon.c.smithi156.stdout: L0 1/0 873.33 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 352.3 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-26T04:46:30.617 INFO:journalctl@ceph.mon.c.smithi156.stdout: L6 1/0 8.28 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T04:46:30.617 INFO:journalctl@ceph.mon.c.smithi156.stdout: Sum 2/0 9.14 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 352.3 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-26T04:46:30.617 INFO:journalctl@ceph.mon.c.smithi156.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 352.3 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-26T04:46:30.617 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-04-26T04:46:30.617 INFO:journalctl@ceph.mon.c.smithi156.stdout: ** Compaction Stats [default] ** 2024-04-26T04:46:30.617 INFO:journalctl@ceph.mon.c.smithi156.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:46:30.617 INFO:journalctl@ceph.mon.c.smithi156.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:46:30.617 INFO:journalctl@ceph.mon.c.smithi156.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 352.3 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-26T04:46:30.617 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-04-26T04:46:30.618 INFO:journalctl@ceph.mon.c.smithi156.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T04:46:30.618 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-04-26T04:46:30.618 INFO:journalctl@ceph.mon.c.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:46:30.618 INFO:journalctl@ceph.mon.c.smithi156.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-26T04:46:30.618 INFO:journalctl@ceph.mon.c.smithi156.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:46:30.618 INFO:journalctl@ceph.mon.c.smithi156.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:46:30.618 INFO:journalctl@ceph.mon.c.smithi156.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:46:30.618 INFO:journalctl@ceph.mon.c.smithi156.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:46:30.618 INFO:journalctl@ceph.mon.c.smithi156.stdout: Cumulative compaction: 0.00 GB write, 124.27 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:46:30.618 INFO:journalctl@ceph.mon.c.smithi156.stdout: Interval compaction: 0.00 GB write, 124.27 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:46:30.618 INFO:journalctl@ceph.mon.c.smithi156.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:46:30.618 INFO:journalctl@ceph.mon.c.smithi156.stdout: Block cache BinnedLRUCache@0x56063e19d090#8 capacity: 512.00 MB usage: 3.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-04-26T04:46:30.618 INFO:journalctl@ceph.mon.c.smithi156.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.03 KB,0.000196695%) IndexBlock(1,2.83 KB,0.000539422%) Misc(1,0.00 KB,0%) 2024-04-26T04:46:30.618 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-04-26T04:46:30.619 INFO:journalctl@ceph.mon.c.smithi156.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:46:30.619 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: starting mon.c rank 2 at public addrs [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] at bind addrs [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:46:30.619 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: mon.c@-1(???) e3 preinit fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:46:30.619 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: mon.c@-1(???).mds e9 new map 2024-04-26T04:46:30.619 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: mon.c@-1(???).mds e9 print_map 2024-04-26T04:46:30.620 INFO:journalctl@ceph.mon.c.smithi156.stdout: e9 2024-04-26T04:46:30.620 INFO:journalctl@ceph.mon.c.smithi156.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:46:30.620 INFO:journalctl@ceph.mon.c.smithi156.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:46:30.620 INFO:journalctl@ceph.mon.c.smithi156.stdout: legacy client fscid: 1 2024-04-26T04:46:30.620 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-04-26T04:46:30.620 INFO:journalctl@ceph.mon.c.smithi156.stdout: Filesystem 'foo' (1) 2024-04-26T04:46:30.620 INFO:journalctl@ceph.mon.c.smithi156.stdout: fs_name foo 2024-04-26T04:46:30.620 INFO:journalctl@ceph.mon.c.smithi156.stdout: epoch 8 2024-04-26T04:46:30.620 INFO:journalctl@ceph.mon.c.smithi156.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-26T04:46:30.621 INFO:journalctl@ceph.mon.c.smithi156.stdout: created 2024-04-26T04:40:44.280108+0000 2024-04-26T04:46:30.621 INFO:journalctl@ceph.mon.c.smithi156.stdout: modified 2024-04-26T04:40:54.847092+0000 2024-04-26T04:46:30.621 INFO:journalctl@ceph.mon.c.smithi156.stdout: tableserver 0 2024-04-26T04:46:30.621 INFO:journalctl@ceph.mon.c.smithi156.stdout: root 0 2024-04-26T04:46:30.622 INFO:journalctl@ceph.mon.c.smithi156.stdout: session_timeout 60 2024-04-26T04:46:30.622 INFO:journalctl@ceph.mon.c.smithi156.stdout: session_autoclose 300 2024-04-26T04:46:30.622 INFO:journalctl@ceph.mon.c.smithi156.stdout: max_file_size 1099511627776 2024-04-26T04:46:30.622 INFO:journalctl@ceph.mon.c.smithi156.stdout: max_xattr_size 65536 2024-04-26T04:46:30.622 INFO:journalctl@ceph.mon.c.smithi156.stdout: required_client_features {} 2024-04-26T04:46:30.622 INFO:journalctl@ceph.mon.c.smithi156.stdout: last_failure 0 2024-04-26T04:46:30.622 INFO:journalctl@ceph.mon.c.smithi156.stdout: last_failure_osd_epoch 0 2024-04-26T04:46:30.623 INFO:journalctl@ceph.mon.c.smithi156.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:46:30.623 INFO:journalctl@ceph.mon.c.smithi156.stdout: max_mds 1 2024-04-26T04:46:30.623 INFO:journalctl@ceph.mon.c.smithi156.stdout: in 0 2024-04-26T04:46:30.623 INFO:journalctl@ceph.mon.c.smithi156.stdout: up {0=14637} 2024-04-26T04:46:30.623 INFO:journalctl@ceph.mon.c.smithi156.stdout: failed 2024-04-26T04:46:30.623 INFO:journalctl@ceph.mon.c.smithi156.stdout: damaged 2024-04-26T04:46:30.623 INFO:journalctl@ceph.mon.c.smithi156.stdout: stopped 2024-04-26T04:46:30.623 INFO:journalctl@ceph.mon.c.smithi156.stdout: data_pools [4] 2024-04-26T04:46:30.623 INFO:journalctl@ceph.mon.c.smithi156.stdout: metadata_pool 3 2024-04-26T04:46:30.623 INFO:journalctl@ceph.mon.c.smithi156.stdout: inline_data disabled 2024-04-26T04:46:30.623 INFO:journalctl@ceph.mon.c.smithi156.stdout: balancer 2024-04-26T04:46:30.623 INFO:journalctl@ceph.mon.c.smithi156.stdout: bal_rank_mask -1 2024-04-26T04:46:30.623 INFO:journalctl@ceph.mon.c.smithi156.stdout: standby_count_wanted 1 2024-04-26T04:46:30.623 INFO:journalctl@ceph.mon.c.smithi156.stdout: qdb_cluster leader: 0 members: 2024-04-26T04:46:30.623 INFO:journalctl@ceph.mon.c.smithi156.stdout: [mds.foo.smithi156.wyfzii{0:14637} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.156:6834/572371201,v1:172.21.15.156:6835/572371201] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:46:30.624 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-04-26T04:46:30.624 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-04-26T04:46:30.624 INFO:journalctl@ceph.mon.c.smithi156.stdout: Standby daemons: 2024-04-26T04:46:30.624 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-04-26T04:46:30.624 INFO:journalctl@ceph.mon.c.smithi156.stdout: [mds.foo.smithi171.kzjlqn{-1:24478} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.171:6832/1083563586,v1:172.21.15.171:6833/1083563586] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:46:30.624 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: mon.c@-1(???).osd e78 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T04:46:30.624 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: mon.c@-1(???).osd e78 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:46:30.624 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: mon.c@-1(???).osd e78 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:46:30.624 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: mon.c@-1(???).osd e78 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:46:30.624 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 ceph-mon[103290]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-04-26T04:46:30.624 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 bash[103251]: 8cac6c96e0db9048064ab747ea8d4ffcc421a7f45968df33dc89a98301a05e07 2024-04-26T04:46:30.625 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:30 smithi156 systemd[1]: Started Ceph mon.c for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:46:35.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:46:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:46:34] "GET /metrics HTTP/1.1" 200 33786 "" "Prometheus/2.43.0" 2024-04-26T04:46:36.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: mon.c calling monitor election 2024-04-26T04:46:36.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: mon.b calling monitor election 2024-04-26T04:46:36.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: mon.a calling monitor election 2024-04-26T04:46:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:46:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: monmap epoch 3 2024-04-26T04:46:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:46:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: last_changed 2024-04-26T04:35:24.769972+0000 2024-04-26T04:46:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: created 2024-04-26T04:33:43.779990+0000 2024-04-26T04:46:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: min_mon_release 17 (quincy) 2024-04-26T04:46:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: election_strategy: 1 2024-04-26T04:46:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-04-26T04:46:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-04-26T04:46:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-04-26T04:46:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:46:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:46:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: mgrmap e38: y(active, since 91s), standbys: x 2024-04-26T04:46:36.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T04:46:36.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:46:36.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:46:36.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[103290]: daemon grafana.a on smithi171 is in error state 2024-04-26T04:46:36.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: mon.c calling monitor election 2024-04-26T04:46:36.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: mon.b calling monitor election 2024-04-26T04:46:36.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: mon.a calling monitor election 2024-04-26T04:46:36.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:46:36.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: monmap epoch 3 2024-04-26T04:46:36.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:46:36.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: last_changed 2024-04-26T04:35:24.769972+0000 2024-04-26T04:46:36.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: created 2024-04-26T04:33:43.779990+0000 2024-04-26T04:46:36.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: min_mon_release 17 (quincy) 2024-04-26T04:46:36.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: election_strategy: 1 2024-04-26T04:46:36.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-04-26T04:46:36.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-04-26T04:46:36.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-04-26T04:46:36.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:46:36.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:46:36.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: mgrmap e38: y(active, since 91s), standbys: x 2024-04-26T04:46:36.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T04:46:36.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:46:36.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:46:36.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:36 smithi156 ceph-mon[98743]: daemon grafana.a on smithi171 is in error state 2024-04-26T04:46:37.002 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: mon.c calling monitor election 2024-04-26T04:46:37.002 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: mon.b calling monitor election 2024-04-26T04:46:37.002 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: mon.a calling monitor election 2024-04-26T04:46:37.002 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:46:37.002 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: monmap epoch 3 2024-04-26T04:46:37.002 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:46:37.002 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: last_changed 2024-04-26T04:35:24.769972+0000 2024-04-26T04:46:37.002 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: created 2024-04-26T04:33:43.779990+0000 2024-04-26T04:46:37.002 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: min_mon_release 17 (quincy) 2024-04-26T04:46:37.002 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: election_strategy: 1 2024-04-26T04:46:37.002 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-04-26T04:46:37.003 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-04-26T04:46:37.003 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-04-26T04:46:37.003 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:46:37.003 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:46:37.003 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: mgrmap e38: y(active, since 91s), standbys: x 2024-04-26T04:46:37.003 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T04:46:37.003 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:46:37.003 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:46:37.003 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:36 smithi171 ceph-mon[31139]: daemon grafana.a on smithi171 is in error state 2024-04-26T04:46:37.205 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-26T04:46:37.832 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:37 smithi156 ceph-mon[103290]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:37.832 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:37 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1921240387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:37.832 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:37 smithi156 ceph-mon[98743]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:37.832 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:37 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1921240387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:37.857 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:46:37.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:37 smithi171 ceph-mon[31139]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:37.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:37 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/1921240387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:38.201 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:38.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:38 smithi156 ceph-mon[98743]: mgrmap e39: y(active, since 92s), standbys: x 2024-04-26T04:46:38.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:38 smithi156 ceph-mon[103290]: mgrmap e39: y(active, since 92s), standbys: x 2024-04-26T04:46:38.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:38 smithi171 ceph-mon[31139]: mgrmap e39: y(active, since 92s), standbys: x 2024-04-26T04:46:39.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:46:38 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:46:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:46:39.661 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:39 smithi156 ceph-mon[98743]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:39.662 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:39 smithi156 ceph-mon[103290]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:39.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:39 smithi171 ceph-mon[31139]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:40.116 INFO:teuthology.orchestra.run.smithi156.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:46:40.116 INFO:teuthology.orchestra.run.smithi156.stdout:alertmanager.a smithi156 *:9093,9094 running (2m) 16s ago 6m 25.7M - 0.25.0 c8568f914cd2 932345b7da32 2024-04-26T04:46:40.116 INFO:teuthology.orchestra.run.smithi156.stdout:grafana.a smithi171 *:3000 error 20s ago 6m - - 2024-04-26T04:46:40.116 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi156.wyfzii smithi156 running (5m) 16s ago 5m 16.1M - 17.2.7-902-gc61472c8 0de407c1b3af f42880bb83a3 2024-04-26T04:46:40.116 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi171.kzjlqn smithi171 running (5m) 20s ago 5m 14.1M - 17.2.7-902-gc61472c8 0de407c1b3af 31b518bc187c 2024-04-26T04:46:40.116 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.x smithi171 *:8443,9283,8765 running (2m) 20s ago 10m 449M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 9ec4d348d704 2024-04-26T04:46:40.117 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.y smithi156 *:8443,9283,8765 running (2m) 16s ago 12m 497M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 715bee4dd354 2024-04-26T04:46:40.117 INFO:teuthology.orchestra.run.smithi156.stdout:mon.a smithi156 running (105s) 16s ago 12m 41.9M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b94e4b5539a1 2024-04-26T04:46:40.117 INFO:teuthology.orchestra.run.smithi156.stdout:mon.b smithi171 running (11m) 20s ago 11m 45.2M 2048M 17.2.7-902-gc61472c8 0de407c1b3af fb359e8b680c 2024-04-26T04:46:40.117 INFO:teuthology.orchestra.run.smithi156.stdout:mon.c smithi156 starting - - - 2048M 2024-04-26T04:46:40.117 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.a smithi156 *:9100 running (2m) 16s ago 6m 16.3M - 1.5.0 0da6a335fe13 5af6998cc7e3 2024-04-26T04:46:40.117 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.b smithi171 *:9100 running (2m) 20s ago 6m 15.1M - 1.5.0 0da6a335fe13 a2d37cfca8cf 2024-04-26T04:46:40.117 INFO:teuthology.orchestra.run.smithi156.stdout:osd.0 smithi156 running (10m) 16s ago 10m 47.7M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e1273966ada0 2024-04-26T04:46:40.117 INFO:teuthology.orchestra.run.smithi156.stdout:osd.1 smithi156 running (10m) 16s ago 10m 47.1M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 7abca656e994 2024-04-26T04:46:40.117 INFO:teuthology.orchestra.run.smithi156.stdout:osd.2 smithi156 running (9m) 16s ago 9m 52.8M 2480M 17.2.7-902-gc61472c8 0de407c1b3af c951e2171c5d 2024-04-26T04:46:40.117 INFO:teuthology.orchestra.run.smithi156.stdout:osd.3 smithi156 running (9m) 16s ago 9m 44.5M 2480M 17.2.7-902-gc61472c8 0de407c1b3af a2c01ca7770f 2024-04-26T04:46:40.117 INFO:teuthology.orchestra.run.smithi156.stdout:osd.4 smithi171 running (9m) 20s ago 9m 50.0M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 388a24a70001 2024-04-26T04:46:40.117 INFO:teuthology.orchestra.run.smithi156.stdout:osd.5 smithi171 running (8m) 20s ago 8m 52.8M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 703235a355b5 2024-04-26T04:46:40.117 INFO:teuthology.orchestra.run.smithi156.stdout:osd.6 smithi171 running (8m) 20s ago 8m 47.0M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 3dad877f8247 2024-04-26T04:46:40.117 INFO:teuthology.orchestra.run.smithi156.stdout:osd.7 smithi171 running (7m) 20s ago 7m 52.6M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e275b2da9b18 2024-04-26T04:46:40.118 INFO:teuthology.orchestra.run.smithi156.stdout:prometheus.a smithi171 *:9095 running (2m) 20s ago 7m 58.9M - 2.43.0 a07b618ecd1d 93d6b22f6fce 2024-04-26T04:46:40.407 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:46:40 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:46:40.310+0000 7f741c426700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-26T04:46:40.835 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T04:46:41.381 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:41 smithi156 ceph-mon[98743]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:41.381 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:41 smithi156 ceph-mon[98743]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:41.381 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:41 smithi156 ceph-mon[103290]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:41.381 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:41 smithi156 ceph-mon[103290]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:41.430 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:41.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:41 smithi171 ceph-mon[31139]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:41.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:41 smithi171 ceph-mon[31139]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:43.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:43 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:43.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:43 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:43.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:43 smithi156 ceph-mon[98743]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:43.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:43 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:43.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:43 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:43.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:43 smithi156 ceph-mon[103290]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:43.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:43 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:43.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:43 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:44.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:43 smithi171 ceph-mon[31139]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:44.108 INFO:teuthology.orchestra.run.smithi156.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:46:44.874 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 60' 2024-04-26T04:46:45.066 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:46:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:46:44] "GET /metrics HTTP/1.1" 200 33780 "" "Prometheus/2.43.0" 2024-04-26T04:46:45.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:45 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:45.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:45 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:45.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:45 smithi156 ceph-mon[98743]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:45.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:45 smithi156 ceph-mon[98743]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:46:45.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:45 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:45.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:45 smithi156 ceph-mon[98743]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:45.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:45 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:45.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:45 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:45.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:45 smithi156 ceph-mon[103290]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:45.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:45 smithi156 ceph-mon[103290]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:46:45.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:45 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:45.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:45 smithi156 ceph-mon[103290]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:45.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:45 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:45.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:45 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:45.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:45 smithi171 ceph-mon[31139]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:45.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:45 smithi171 ceph-mon[31139]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:46:45.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:45 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:45.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:45 smithi171 ceph-mon[31139]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:46.331 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:47.920 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:47 smithi156 ceph-mon[98743]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:47.920 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:47 smithi156 ceph-mon[103290]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:48.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:47 smithi171 ceph-mon[31139]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T04:46:48.999 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:46:48 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:46:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:46:49.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:49 smithi171 ceph-mon[31139]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:50.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:49 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:50.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:49 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:50.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:49 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:50.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:49 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:50.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:49 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:50.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:49 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:50.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:49 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:50.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:49 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:50.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:49 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:50.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[103290]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:50.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:50.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:50.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:50.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:50.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:50.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:50.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:50.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:50.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:50.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[98743]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:50.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:50.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:50.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:50.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:50.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:50.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:50.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:50.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:50.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:49 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:46:50.499 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:46:50 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:46:50.086Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", public_addr=\"172.21.15.156\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", public_addr=\"172.21.15.156\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-04-26T04:46:50.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:50 smithi171 ceph-mon[31139]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:51.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:50 smithi171 ceph-mon[31139]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:51.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:50 smithi171 ceph-mon[31139]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:51.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:50 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:51.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:50 smithi156 ceph-mon[98743]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:51.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:50 smithi156 ceph-mon[98743]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:51.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:50 smithi156 ceph-mon[98743]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:51.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:50 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:51.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:50 smithi156 ceph-mon[103290]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:51.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:50 smithi156 ceph-mon[103290]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:51.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:50 smithi156 ceph-mon[103290]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:51.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:50 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:52.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:52 smithi171 ceph-mon[31139]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:53.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:52 smithi156 ceph-mon[103290]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:53.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:52 smithi156 ceph-mon[98743]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:55.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:54 smithi171 ceph-mon[31139]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:55.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:54 smithi156 ceph-mon[98743]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:55.094 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:46:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:46:54] "GET /metrics HTTP/1.1" 200 33781 "" "Prometheus/2.43.0" 2024-04-26T04:46:55.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:54 smithi156 ceph-mon[103290]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:56.989 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:56 smithi171 ceph-mon[31139]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:57.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:56 smithi156 ceph-mon[98743]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:57.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:56 smithi156 ceph-mon[103290]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:58.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:46:58 smithi171 ceph-mon[31139]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:59.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:46:58 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:46:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:46:59.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:46:58 smithi156 ceph-mon[98743]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:46:59.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:46:58 smithi156 ceph-mon[103290]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:00.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:00 smithi171 ceph-mon[31139]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:01.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:00 smithi156 ceph-mon[98743]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:01.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:00 smithi156 ceph-mon[103290]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:02.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:02 smithi171 ceph-mon[31139]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:03.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:02 smithi156 ceph-mon[103290]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:03.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:02 smithi156 ceph-mon[98743]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:04.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:04 smithi171 ceph-mon[31139]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:05.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:04 smithi156 ceph-mon[98743]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:05.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:47:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:47:04] "GET /metrics HTTP/1.1" 200 33781 "" "Prometheus/2.43.0" 2024-04-26T04:47:05.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:04 smithi156 ceph-mon[103290]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:05.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:05 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:06.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:05 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:06.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:06.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:06 smithi171 ceph-mon[31139]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:07.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:06 smithi156 ceph-mon[103290]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:07.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:06 smithi156 ceph-mon[98743]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:08.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:08 smithi171 ceph-mon[31139]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:09.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:47:08 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:47:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:47:09.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:08 smithi156 ceph-mon[98743]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:09.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:08 smithi156 ceph-mon[103290]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:10.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:10 smithi171 ceph-mon[31139]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:11.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:10 smithi156 ceph-mon[98743]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:11.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:10 smithi156 ceph-mon[103290]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:12.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:12 smithi171 ceph-mon[31139]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:13.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:12 smithi156 ceph-mon[98743]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:13.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:12 smithi156 ceph-mon[103290]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:14.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:14 smithi171 ceph-mon[31139]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:15.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:47:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:47:14] "GET /metrics HTTP/1.1" 200 33781 "" "Prometheus/2.43.0" 2024-04-26T04:47:15.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:14 smithi156 ceph-mon[98743]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:15.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:14 smithi156 ceph-mon[103290]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:16.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:16 smithi171 ceph-mon[31139]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:17.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:16 smithi156 ceph-mon[98743]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:17.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:16 smithi156 ceph-mon[103290]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:18.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:18 smithi171 ceph-mon[31139]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:19.001 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:47:18 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:47:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:47:19.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:18 smithi156 ceph-mon[98743]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:19.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:18 smithi156 ceph-mon[103290]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:20.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:20 smithi171 ceph-mon[31139]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:20.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:20 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:21.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:20 smithi156 ceph-mon[98743]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:21.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:21.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:20 smithi156 ceph-mon[103290]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:21.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:22.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:22 smithi171 ceph-mon[31139]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:23.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:22 smithi156 ceph-mon[98743]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:23.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:22 smithi156 ceph-mon[103290]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:25.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:24 smithi171 ceph-mon[31139]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:25.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:24 smithi156 ceph-mon[98743]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:25.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:47:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:47:24] "GET /metrics HTTP/1.1" 200 33719 "" "Prometheus/2.43.0" 2024-04-26T04:47:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:24 smithi156 ceph-mon[103290]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:26.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:26 smithi171 ceph-mon[31139]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:27.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:26 smithi156 ceph-mon[98743]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:27.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:26 smithi156 ceph-mon[103290]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:28.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:28 smithi171 ceph-mon[31139]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:29.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:47:28 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:47:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:47:29.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:28 smithi156 ceph-mon[98743]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:29.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:28 smithi156 ceph-mon[103290]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:30.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:30 smithi171 ceph-mon[31139]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:31.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:30 smithi156 ceph-mon[98743]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:31.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:30 smithi156 ceph-mon[103290]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:32.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:32 smithi171 ceph-mon[31139]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:33.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:32 smithi156 ceph-mon[103290]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:33.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:32 smithi156 ceph-mon[98743]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:34.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:34 smithi171 ceph-mon[31139]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:35.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:34 smithi156 ceph-mon[103290]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:35.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:34 smithi156 ceph-mon[98743]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:35.094 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:47:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:47:34] "GET /metrics HTTP/1.1" 200 33719 "" "Prometheus/2.43.0" 2024-04-26T04:47:35.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:35 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:36.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:36.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:36.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:36 smithi171 ceph-mon[31139]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:37.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:36 smithi156 ceph-mon[98743]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:37.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:36 smithi156 ceph-mon[103290]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:38.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:38 smithi171 ceph-mon[31139]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:39.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:47:38 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:47:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:47:39.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:38 smithi156 ceph-mon[98743]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:39.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:38 smithi156 ceph-mon[103290]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:40.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:40 smithi171 ceph-mon[31139]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:41.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:40 smithi156 ceph-mon[98743]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:41.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:40 smithi156 ceph-mon[103290]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:42.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:42 smithi171 ceph-mon[31139]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:43.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:42 smithi156 ceph-mon[98743]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:43.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:42 smithi156 ceph-mon[103290]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:44.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:44 smithi171 ceph-mon[31139]: pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:45.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:44 smithi156 ceph-mon[98743]: pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:45.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:47:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:47:44] "GET /metrics HTTP/1.1" 200 33790 "" "Prometheus/2.43.0" 2024-04-26T04:47:45.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:44 smithi156 ceph-mon[103290]: pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:46.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:46 smithi171 ceph-mon[31139]: pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:47.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:46 smithi156 ceph-mon[98743]: pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:47.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:46 smithi156 ceph-mon[103290]: pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:49.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:48 smithi171 ceph-mon[31139]: pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:49.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:47:48 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:47:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:47:49.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:48 smithi156 ceph-mon[103290]: pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:49.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:48 smithi156 ceph-mon[98743]: pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:49.156 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade resume' 2024-04-26T04:47:49.417 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:47:49.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:49 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:47:49.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:49 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:47:49.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:49 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:47:50.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:50 smithi171 ceph-mon[31139]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:50.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:50 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:51.019 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:50 smithi156 ceph-mon[103290]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:51.019 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:50 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:51.019 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:50 smithi156 ceph-mon[98743]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:51.019 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:50 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:51.509 INFO:teuthology.orchestra.run.smithi156.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:47:52.167 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for all mons to upgrade' 2024-04-26T04:47:52.657 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:47:52.782 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:52 smithi156 ceph-mon[98743]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:47:52.782 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:52 smithi156 ceph-mon[98743]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:47:52.782 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:52 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:47:52.782 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:52 smithi156 ceph-mon[98743]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:52.783 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:52 smithi156 ceph-mon[103290]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:47:52.783 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:52 smithi156 ceph-mon[103290]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:47:52.783 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:52 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:47:52.783 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:52 smithi156 ceph-mon[103290]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:52.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:52 smithi171 ceph-mon[31139]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:47:53.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:52 smithi171 ceph-mon[31139]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:47:53.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:52 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:47:53.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:52 smithi171 ceph-mon[31139]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:54.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:54 smithi171 ceph-mon[31139]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:54.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:54 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:47:54.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:54 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:47:55.016 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:47:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:47:54] "GET /metrics HTTP/1.1" 200 33790 "" "Prometheus/2.43.0" 2024-04-26T04:47:55.016 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:54 smithi156 ceph-mon[98743]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:55.016 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:54 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:47:55.016 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:54 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:47:55.017 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:54 smithi156 ceph-mon[103290]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:55.017 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:54 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:47:55.017 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:54 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:47:55.132 INFO:teuthology.orchestra.run.smithi156.stdout:wait for all mons to upgrade 2024-04-26T04:47:55.763 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-04-26T04:47:56.532 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:47:57.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:56 smithi171 ceph-mon[31139]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:57.045 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:56 smithi156 ceph-mon[98743]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:57.045 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:56 smithi156 ceph-mon[103290]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:58.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:58 smithi171 ceph-mon[31139]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:59.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:47:58 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:47:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:47:59.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:58 smithi156 ceph-mon[98743]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:47:59.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:58 smithi156 ceph-mon[103290]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:00.124 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:59 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:00.124 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:59 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:00.124 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:59 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3079756061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:00.124 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:59 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:00.124 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:59 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:00.124 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:59 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:00.124 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:59 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:00.124 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:59 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:48:00.124 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:59 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:48:00.124 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:47:59 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T04:48:00.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:00.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:00.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3079756061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:00.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:00.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:00.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:00.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:00.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:48:00.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:48:00.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T04:48:00.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:00.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:00.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3079756061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:00.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:00.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:00.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:00.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:00.347 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:48:00.347 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:48:00.347 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:47:59 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T04:48:01.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:00 smithi171 ceph-mon[31139]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:01.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:00 smithi171 ceph-mon[31139]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:01.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:00 smithi171 ceph-mon[31139]: Upgrade: It appears safe to stop mon.b 2024-04-26T04:48:01.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:00 smithi156 ceph-mon[98743]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:01.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:00 smithi156 ceph-mon[98743]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:01.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:00 smithi156 ceph-mon[98743]: Upgrade: It appears safe to stop mon.b 2024-04-26T04:48:01.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:00 smithi156 ceph-mon[103290]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:01.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:00 smithi156 ceph-mon[103290]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:01.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:00 smithi156 ceph-mon[103290]: Upgrade: It appears safe to stop mon.b 2024-04-26T04:48:02.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:02 smithi171 ceph-mon[31139]: Upgrade: Updating mon.b 2024-04-26T04:48:02.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:02 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:02.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:02 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:48:02.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:02 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:48:02.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:02 smithi171 ceph-mon[31139]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:02.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:02 smithi171 ceph-mon[31139]: Deploying daemon mon.b on smithi171 2024-04-26T04:48:02.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:02 smithi171 ceph-mon[31139]: from='client.? 172.21.15.156:0/3292951478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:02.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:02 smithi171 ceph-mon[31139]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:02.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:02 smithi156 ceph-mon[98743]: Upgrade: Updating mon.b 2024-04-26T04:48:02.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:02 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:02.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:02 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:48:02.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:02 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:48:02.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:02 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:02.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:02 smithi156 ceph-mon[98743]: Deploying daemon mon.b on smithi171 2024-04-26T04:48:02.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:02 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3292951478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:02.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:02 smithi156 ceph-mon[98743]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:02.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:02 smithi156 ceph-mon[103290]: Upgrade: Updating mon.b 2024-04-26T04:48:02.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:02 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:02.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:02 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:48:02.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:02 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:48:02.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:02 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:02.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:02 smithi156 ceph-mon[103290]: Deploying daemon mon.b on smithi171 2024-04-26T04:48:02.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:02 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3292951478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:02.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:02 smithi156 ceph-mon[103290]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:03.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:03 smithi171 systemd[1]: Stopping Ceph mon.b for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:48:03.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-b[31115]: 2024-04-26T04:48:03.481+0000 7fabf1344700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:48:03.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:03 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-b[31115]: 2024-04-26T04:48:03.481+0000 7fabf1344700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-04-26T04:48:03.946 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:03 smithi171 podman[90374]: 2024-04-26 04:48:03.655429077 +0000 UTC m=+0.276548008 container died fb359e8b680cea83c4e9f9b6a566897361752fcd27c19510a16d0c8e0ec7aeec (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-b, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS) 2024-04-26T04:48:04.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:04 smithi171 podman[90374]: 2024-04-26 04:48:04.045092198 +0000 UTC m=+0.666211128 container cleanup fb359e8b680cea83c4e9f9b6a566897361752fcd27c19510a16d0c8e0ec7aeec (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-b, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True) 2024-04-26T04:48:04.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:04 smithi171 bash[90374]: ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-b 2024-04-26T04:48:04.612 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:04 smithi171 podman[90393]: 2024-04-26 04:48:04.287981659 +0000 UTC m=+0.635644465 container remove fb359e8b680cea83c4e9f9b6a566897361752fcd27c19510a16d0c8e0ec7aeec (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424) 2024-04-26T04:48:04.612 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:04 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.b.service: Deactivated successfully. 2024-04-26T04:48:04.612 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:04 smithi171 systemd[1]: Stopped Ceph mon.b for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:48:04.612 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:04 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.b.service: Consumed 9.083s CPU time. 2024-04-26T04:48:04.965 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:04 smithi171 systemd[1]: Starting Ceph mon.b for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:48:04.965 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:04 smithi171 podman[90529]: 2024-04-26 04:48:04.731173235 +0000 UTC m=+0.018165892 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:48:04.965 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:04 smithi171 podman[90529]: 2024-04-26 04:48:04.838971268 +0000 UTC m=+0.125963904 container create 0829e2e150f0f5afd77de2d85eb01e6b1a0a409df8283ffa2464b9d1fc04d867 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-b, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS) 2024-04-26T04:48:05.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:48:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:48:04] "GET /metrics HTTP/1.1" 200 33790 "" "Prometheus/2.43.0" 2024-04-26T04:48:05.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 podman[90529]: 2024-04-26 04:48:05.041421123 +0000 UTC m=+0.328413774 container init 0829e2e150f0f5afd77de2d85eb01e6b1a0a409df8283ffa2464b9d1fc04d867 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-b, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=) 2024-04-26T04:48:05.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 podman[90529]: 2024-04-26 04:48:05.046709681 +0000 UTC m=+0.333702320 container start 0829e2e150f0f5afd77de2d85eb01e6b1a0a409df8283ffa2464b9d1fc04d867 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424) 2024-04-26T04:48:05.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:48:05.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 7 2024-04-26T04:48:05.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: pidfile_write: ignore empty --pid-file 2024-04-26T04:48:05.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: load: jerasure load: lrc 2024-04-26T04:48:05.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: RocksDB version: 7.9.2 2024-04-26T04:48:05.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Git sha 0 2024-04-26T04:48:05.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-26T04:48:05.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: DB SUMMARY 2024-04-26T04:48:05.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: DB Session ID: Q0JRNMABQBRGG8ELKRAE 2024-04-26T04:48:05.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:48:05.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:48:05.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1084 Bytes 2024-04-26T04:48:05.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-04-26T04:48:05.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 3243374 ; 2024-04-26T04:48:05.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:48:05.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:48:05.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:48:05.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T04:48:05.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:48:05.502 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T04:48:05.502 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.env: 0x55c1405d2c60 2024-04-26T04:48:05.502 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T04:48:05.502 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.info_log: 0x55c14287daa0 2024-04-26T04:48:05.502 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:48:05.502 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.statistics: (nil) 2024-04-26T04:48:05.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.use_fsync: 0 2024-04-26T04:48:05.504 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:48:05.504 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:48:05.504 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:48:05.504 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:48:05.504 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:48:05.504 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:48:05.504 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:48:05.504 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:48:05.504 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:48:05.504 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:48:05.504 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:48:05.504 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.db_log_dir: 2024-04-26T04:48:05.504 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.wal_dir: 2024-04-26T04:48:05.505 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:48:05.505 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:48:05.505 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:48:05.505 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:48:05.505 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:48:05.505 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:48:05.505 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:48:05.505 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:48:05.505 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.write_buffer_manager: 0x55c142850820 2024-04-26T04:48:05.505 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:48:05.505 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:48:05.505 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:48:05.505 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:48:05.505 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:48:05.506 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:48:05.506 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:48:05.506 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:48:05.506 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.unordered_write: 0 2024-04-26T04:48:05.506 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:48:05.506 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:48:05.506 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:48:05.506 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:48:05.506 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.row_cache: None 2024-04-26T04:48:05.506 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.wal_filter: None 2024-04-26T04:48:05.506 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:48:05.506 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:48:05.506 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:48:05.506 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:48:05.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.wal_compression: 0 2024-04-26T04:48:05.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:48:05.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:48:05.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:48:05.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:48:05.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:48:05.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:48:05.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:48:05.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:48:05.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:48:05.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:48:05.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:48:05.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T04:48:05.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:48:05.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:48:05.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:48:05.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:48:05.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:48:05.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:48:05.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:48:05.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:48:05.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:48:05.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:48:05.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:48:05.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_open_files: -1 2024-04-26T04:48:05.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:48:05.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:48:05.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:48:05.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:48:05.509 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:48:05.509 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Compression algorithms supported: 2024-04-26T04:48:05.509 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:48:05.509 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: kZSTD supported: 0 2024-04-26T04:48:05.509 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:48:05.509 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:48:05.509 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:48:05.509 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:48:05.509 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:48:05.509 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:48:05.509 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:48:05.509 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T04:48:05.509 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-04-26T04:48:05.509 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T04:48:05.510 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:48:05.510 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.merge_operator: 2024-04-26T04:48:05.510 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compaction_filter: None 2024-04-26T04:48:05.510 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:48:05.510 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:48:05.510 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:48:05.510 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:48:05.510 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c14287dba0) 2024-04-26T04:48:05.510 INFO:journalctl@ceph.mon.b.smithi171.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:48:05.510 INFO:journalctl@ceph.mon.b.smithi171.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:48:05.510 INFO:journalctl@ceph.mon.b.smithi171.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:48:05.510 INFO:journalctl@ceph.mon.b.smithi171.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:48:05.510 INFO:journalctl@ceph.mon.b.smithi171.stdout: index_type: 0 2024-04-26T04:48:05.510 INFO:journalctl@ceph.mon.b.smithi171.stdout: data_block_index_type: 0 2024-04-26T04:48:05.510 INFO:journalctl@ceph.mon.b.smithi171.stdout: index_shortening: 1 2024-04-26T04:48:05.511 INFO:journalctl@ceph.mon.b.smithi171.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:48:05.511 INFO:journalctl@ceph.mon.b.smithi171.stdout: checksum: 4 2024-04-26T04:48:05.511 INFO:journalctl@ceph.mon.b.smithi171.stdout: no_block_cache: 0 2024-04-26T04:48:05.511 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_cache: 0x55c142881090 2024-04-26T04:48:05.511 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:48:05.511 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_cache_options: 2024-04-26T04:48:05.511 INFO:journalctl@ceph.mon.b.smithi171.stdout: capacity : 536870912 2024-04-26T04:48:05.511 INFO:journalctl@ceph.mon.b.smithi171.stdout: num_shard_bits : 4 2024-04-26T04:48:05.511 INFO:journalctl@ceph.mon.b.smithi171.stdout: strict_capacity_limit : 0 2024-04-26T04:48:05.511 INFO:journalctl@ceph.mon.b.smithi171.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:48:05.511 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_cache_compressed: (nil) 2024-04-26T04:48:05.511 INFO:journalctl@ceph.mon.b.smithi171.stdout: persistent_cache: (nil) 2024-04-26T04:48:05.511 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_size: 4096 2024-04-26T04:48:05.511 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_size_deviation: 10 2024-04-26T04:48:05.511 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_restart_interval: 16 2024-04-26T04:48:05.512 INFO:journalctl@ceph.mon.b.smithi171.stdout: index_block_restart_interval: 1 2024-04-26T04:48:05.512 INFO:journalctl@ceph.mon.b.smithi171.stdout: metadata_block_size: 4096 2024-04-26T04:48:05.512 INFO:journalctl@ceph.mon.b.smithi171.stdout: partition_filters: 0 2024-04-26T04:48:05.512 INFO:journalctl@ceph.mon.b.smithi171.stdout: use_delta_encoding: 1 2024-04-26T04:48:05.512 INFO:journalctl@ceph.mon.b.smithi171.stdout: filter_policy: bloomfilter 2024-04-26T04:48:05.512 INFO:journalctl@ceph.mon.b.smithi171.stdout: whole_key_filtering: 1 2024-04-26T04:48:05.512 INFO:journalctl@ceph.mon.b.smithi171.stdout: verify_compression: 0 2024-04-26T04:48:05.512 INFO:journalctl@ceph.mon.b.smithi171.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:48:05.512 INFO:journalctl@ceph.mon.b.smithi171.stdout: format_version: 5 2024-04-26T04:48:05.512 INFO:journalctl@ceph.mon.b.smithi171.stdout: enable_index_compression: 1 2024-04-26T04:48:05.512 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_align: 0 2024-04-26T04:48:05.512 INFO:journalctl@ceph.mon.b.smithi171.stdout: max_auto_readahead_size: 262144 2024-04-26T04:48:05.512 INFO:journalctl@ceph.mon.b.smithi171.stdout: prepopulate_block_cache: 0 2024-04-26T04:48:05.512 INFO:journalctl@ceph.mon.b.smithi171.stdout: initial_auto_readahead_size: 8192 2024-04-26T04:48:05.512 INFO:journalctl@ceph.mon.b.smithi171.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T04:48:05.513 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:48:05.513 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:48:05.513 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compression: NoCompression 2024-04-26T04:48:05.513 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:48:05.513 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:48:05.513 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:48:05.513 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.num_levels: 7 2024-04-26T04:48:05.513 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:48:05.513 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:48:05.513 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:48:05.513 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:48:05.513 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:48:05.513 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:48:05.513 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:48:05.513 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:48:05.514 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:48:05.514 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:48:05.514 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:48:05.514 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:48:05.514 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:48:05.514 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:48:05.514 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:48:05.514 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:48:05.514 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:48:05.514 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:48:05.514 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:48:05.514 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:48:05.514 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:48:05.514 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:48:05.515 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:48:05.515 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:48:05.515 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:48:05.515 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:48:05.515 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:48:05.515 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:48:05.515 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:48:05.515 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:48:05.515 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:48:05.515 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:48:05.515 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:48:05.515 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:48:05.515 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:48:05.515 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:48:05.515 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:48:05.516 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:48:05.516 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T04:48:05.516 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T04:48:05.516 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:48:05.516 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:48:05.516 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:48:05.516 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:48:05.516 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:48:05.516 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:48:05.516 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:48:05.516 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:48:05.516 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:48:05.516 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:48:05.516 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.ttl: 2592000 2024-04-26T04:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:48:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T04:48:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T04:48:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:48:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:48:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:48:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:48:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:48:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:48:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T04:48:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T04:48:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T04:48:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T04:48:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-04-26T04:48:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:48:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12996, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:48:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-26T04:48:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 8ccd95c9-732a-43bb-be23-80f8cb0299ba 2024-04-26T04:48:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106885076550, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-26T04:48:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-26T04:48:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106885084946, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 1966632, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13001, "largest_seqno": 15000, "table_properties": {"data_size": 1958357, "index_size": 5468, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1797, "raw_key_size": 15357, "raw_average_key_size": 21, "raw_value_size": 1942594, "raw_average_value_size": 2736, "num_data_blocks": 252, "num_entries": 710, "num_filter_entries": 710, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714106885, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "8ccd95c9-732a-43bb-be23-80f8cb0299ba", "db_session_id": "Q0JRNMABQBRGG8ELKRAE", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-26T04:48:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106885085077, "job": 1, "event": "recovery_finished"} 2024-04-26T04:48:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-26T04:48:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:48:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:48:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55c142972000 2024-04-26T04:48:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: DB pointer 0x55c14295c000 2024-04-26T04:48:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T04:48:05.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T04:48:05.520 INFO:journalctl@ceph.mon.b.smithi171.stdout: ** DB Stats ** 2024-04-26T04:48:05.520 INFO:journalctl@ceph.mon.b.smithi171.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:48:05.520 INFO:journalctl@ceph.mon.b.smithi171.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:48:05.520 INFO:journalctl@ceph.mon.b.smithi171.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:48:05.520 INFO:journalctl@ceph.mon.b.smithi171.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:48:05.520 INFO:journalctl@ceph.mon.b.smithi171.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:48:05.520 INFO:journalctl@ceph.mon.b.smithi171.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:48:05.520 INFO:journalctl@ceph.mon.b.smithi171.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:48:05.520 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-04-26T04:48:05.520 INFO:journalctl@ceph.mon.b.smithi171.stdout: ** Compaction Stats [default] ** 2024-04-26T04:48:05.520 INFO:journalctl@ceph.mon.b.smithi171.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:48:05.520 INFO:journalctl@ceph.mon.b.smithi171.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T04:48:05.520 INFO:journalctl@ceph.mon.b.smithi171.stdout: L0 1/0 1.88 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 549.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-26T04:48:05.520 INFO:journalctl@ceph.mon.b.smithi171.stdout: L6 1/0 8.28 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T04:48:05.521 INFO:journalctl@ceph.mon.b.smithi171.stdout: Sum 2/0 10.16 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 549.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-26T04:48:05.521 INFO:journalctl@ceph.mon.b.smithi171.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 549.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-26T04:48:05.521 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-04-26T04:48:05.521 INFO:journalctl@ceph.mon.b.smithi171.stdout: ** Compaction Stats [default] ** 2024-04-26T04:48:05.521 INFO:journalctl@ceph.mon.b.smithi171.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:48:05.521 INFO:journalctl@ceph.mon.b.smithi171.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:48:05.521 INFO:journalctl@ceph.mon.b.smithi171.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 549.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-26T04:48:05.521 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-04-26T04:48:05.521 INFO:journalctl@ceph.mon.b.smithi171.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T04:48:05.521 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-04-26T04:48:05.521 INFO:journalctl@ceph.mon.b.smithi171.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:48:05.521 INFO:journalctl@ceph.mon.b.smithi171.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-26T04:48:05.521 INFO:journalctl@ceph.mon.b.smithi171.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:48:05.521 INFO:journalctl@ceph.mon.b.smithi171.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:48:05.521 INFO:journalctl@ceph.mon.b.smithi171.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:48:05.522 INFO:journalctl@ceph.mon.b.smithi171.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:48:05.522 INFO:journalctl@ceph.mon.b.smithi171.stdout: Cumulative compaction: 0.00 GB write, 151.25 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:48:05.522 INFO:journalctl@ceph.mon.b.smithi171.stdout: Interval compaction: 0.00 GB write, 151.25 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:48:05.522 INFO:journalctl@ceph.mon.b.smithi171.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:48:05.522 INFO:journalctl@ceph.mon.b.smithi171.stdout: Block cache BinnedLRUCache@0x55c142881090#7 capacity: 512.00 MB usage: 8.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-04-26T04:48:05.522 INFO:journalctl@ceph.mon.b.smithi171.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2024-04-26T04:48:05.522 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-04-26T04:48:05.522 INFO:journalctl@ceph.mon.b.smithi171.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:48:05.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: starting mon.b rank 1 at public addrs [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] at bind addrs [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:48:05.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: mon.b@-1(???) e3 preinit fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:48:05.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: mon.b@-1(???).mds e9 new map 2024-04-26T04:48:05.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: mon.b@-1(???).mds e9 print_map 2024-04-26T04:48:05.522 INFO:journalctl@ceph.mon.b.smithi171.stdout: e9 2024-04-26T04:48:05.522 INFO:journalctl@ceph.mon.b.smithi171.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:48:05.523 INFO:journalctl@ceph.mon.b.smithi171.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:48:05.523 INFO:journalctl@ceph.mon.b.smithi171.stdout: legacy client fscid: 1 2024-04-26T04:48:05.523 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-04-26T04:48:05.523 INFO:journalctl@ceph.mon.b.smithi171.stdout: Filesystem 'foo' (1) 2024-04-26T04:48:05.523 INFO:journalctl@ceph.mon.b.smithi171.stdout: fs_name foo 2024-04-26T04:48:05.523 INFO:journalctl@ceph.mon.b.smithi171.stdout: epoch 8 2024-04-26T04:48:05.523 INFO:journalctl@ceph.mon.b.smithi171.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-26T04:48:05.523 INFO:journalctl@ceph.mon.b.smithi171.stdout: created 2024-04-26T04:40:44.280108+0000 2024-04-26T04:48:05.523 INFO:journalctl@ceph.mon.b.smithi171.stdout: modified 2024-04-26T04:40:54.847092+0000 2024-04-26T04:48:05.523 INFO:journalctl@ceph.mon.b.smithi171.stdout: tableserver 0 2024-04-26T04:48:05.523 INFO:journalctl@ceph.mon.b.smithi171.stdout: root 0 2024-04-26T04:48:05.523 INFO:journalctl@ceph.mon.b.smithi171.stdout: session_timeout 60 2024-04-26T04:48:05.523 INFO:journalctl@ceph.mon.b.smithi171.stdout: session_autoclose 300 2024-04-26T04:48:05.523 INFO:journalctl@ceph.mon.b.smithi171.stdout: max_file_size 1099511627776 2024-04-26T04:48:05.523 INFO:journalctl@ceph.mon.b.smithi171.stdout: max_xattr_size 65536 2024-04-26T04:48:05.523 INFO:journalctl@ceph.mon.b.smithi171.stdout: required_client_features {} 2024-04-26T04:48:05.524 INFO:journalctl@ceph.mon.b.smithi171.stdout: last_failure 0 2024-04-26T04:48:05.524 INFO:journalctl@ceph.mon.b.smithi171.stdout: last_failure_osd_epoch 0 2024-04-26T04:48:05.524 INFO:journalctl@ceph.mon.b.smithi171.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:48:05.524 INFO:journalctl@ceph.mon.b.smithi171.stdout: max_mds 1 2024-04-26T04:48:05.524 INFO:journalctl@ceph.mon.b.smithi171.stdout: in 0 2024-04-26T04:48:05.524 INFO:journalctl@ceph.mon.b.smithi171.stdout: up {0=14637} 2024-04-26T04:48:05.524 INFO:journalctl@ceph.mon.b.smithi171.stdout: failed 2024-04-26T04:48:05.524 INFO:journalctl@ceph.mon.b.smithi171.stdout: damaged 2024-04-26T04:48:05.524 INFO:journalctl@ceph.mon.b.smithi171.stdout: stopped 2024-04-26T04:48:05.524 INFO:journalctl@ceph.mon.b.smithi171.stdout: data_pools [4] 2024-04-26T04:48:05.524 INFO:journalctl@ceph.mon.b.smithi171.stdout: metadata_pool 3 2024-04-26T04:48:05.524 INFO:journalctl@ceph.mon.b.smithi171.stdout: inline_data disabled 2024-04-26T04:48:05.524 INFO:journalctl@ceph.mon.b.smithi171.stdout: balancer 2024-04-26T04:48:05.524 INFO:journalctl@ceph.mon.b.smithi171.stdout: bal_rank_mask -1 2024-04-26T04:48:05.525 INFO:journalctl@ceph.mon.b.smithi171.stdout: standby_count_wanted 1 2024-04-26T04:48:05.525 INFO:journalctl@ceph.mon.b.smithi171.stdout: qdb_cluster leader: 0 members: 2024-04-26T04:48:05.525 INFO:journalctl@ceph.mon.b.smithi171.stdout: [mds.foo.smithi156.wyfzii{0:14637} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.156:6834/572371201,v1:172.21.15.156:6835/572371201] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:48:05.525 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-04-26T04:48:05.525 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-04-26T04:48:05.525 INFO:journalctl@ceph.mon.b.smithi171.stdout: Standby daemons: 2024-04-26T04:48:05.525 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-04-26T04:48:05.525 INFO:journalctl@ceph.mon.b.smithi171.stdout: [mds.foo.smithi171.kzjlqn{-1:24478} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.171:6832/1083563586,v1:172.21.15.171:6833/1083563586] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:48:05.525 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: mon.b@-1(???).osd e78 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T04:48:05.525 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: mon.b@-1(???).osd e78 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:48:05.525 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: mon.b@-1(???).osd e78 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:48:05.525 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: mon.b@-1(???).osd e78 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:48:05.525 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 ceph-mon[90566]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-04-26T04:48:05.525 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 bash[90529]: 0829e2e150f0f5afd77de2d85eb01e6b1a0a409df8283ffa2464b9d1fc04d867 2024-04-26T04:48:05.526 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:05 smithi171 systemd[1]: Started Ceph mon.b for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:48:06.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:06 smithi156 ceph-mon[103290]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:06.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:06 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3527413602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:06.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:06 smithi156 ceph-mon[98743]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:06.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:06 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3527413602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:07.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:06 smithi171 ceph-mon[90566]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:07.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:06 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3527413602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:09.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:48:08 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:48:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:48:10.447 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:48:10 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:48:10.095+0000 7f741c426700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-26T04:48:11.859 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-26T04:48:12.464 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:48:12.701 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: mon.c calling monitor election 2024-04-26T04:48:12.701 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: mon.b calling monitor election 2024-04-26T04:48:12.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:12.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: last_changed 2024-04-26T04:48:06.508401+0000 2024-04-26T04:48:12.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: created 2024-04-26T04:33:43.779990+0000 2024-04-26T04:48:12.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: min_mon_release 19 (squid) 2024-04-26T04:48:12.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: election_strategy: 1 2024-04-26T04:48:12.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-04-26T04:48:12.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-04-26T04:48:12.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-04-26T04:48:12.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:48:12.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:48:12.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: mgrmap e39: y(active, since 3m), standbys: x 2024-04-26T04:48:12.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-26T04:48:12.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:48:12.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:48:12.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: daemon grafana.a on smithi171 is in error state 2024-04-26T04:48:12.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: mon.a calling monitor election 2024-04-26T04:48:12.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:48:12.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: monmap epoch 4 2024-04-26T04:48:12.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:48:12.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: last_changed 2024-04-26T04:48:06.508401+0000 2024-04-26T04:48:12.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: created 2024-04-26T04:33:43.779990+0000 2024-04-26T04:48:12.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: min_mon_release 19 (squid) 2024-04-26T04:48:12.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: election_strategy: 1 2024-04-26T04:48:12.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-04-26T04:48:12.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-04-26T04:48:12.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-04-26T04:48:12.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:48:12.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:48:12.704 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: mgrmap e39: y(active, since 3m), standbys: x 2024-04-26T04:48:12.704 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-26T04:48:12.704 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:48:12.704 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:48:12.704 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: daemon grafana.a on smithi171 is in error state 2024-04-26T04:48:12.704 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:12.704 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:12.704 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:12.704 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1205962104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:12.704 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: mon.c calling monitor election 2024-04-26T04:48:12.704 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: mon.b calling monitor election 2024-04-26T04:48:12.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:12.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: last_changed 2024-04-26T04:48:06.508401+0000 2024-04-26T04:48:12.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: created 2024-04-26T04:33:43.779990+0000 2024-04-26T04:48:12.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: min_mon_release 19 (squid) 2024-04-26T04:48:12.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: election_strategy: 1 2024-04-26T04:48:12.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-04-26T04:48:12.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-04-26T04:48:12.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-04-26T04:48:12.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:48:12.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:48:12.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: mgrmap e39: y(active, since 3m), standbys: x 2024-04-26T04:48:12.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-26T04:48:12.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:48:12.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:48:12.706 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: daemon grafana.a on smithi171 is in error state 2024-04-26T04:48:12.706 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: mon.a calling monitor election 2024-04-26T04:48:12.706 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:48:12.706 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: monmap epoch 4 2024-04-26T04:48:12.706 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:48:12.706 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: last_changed 2024-04-26T04:48:06.508401+0000 2024-04-26T04:48:12.706 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: created 2024-04-26T04:33:43.779990+0000 2024-04-26T04:48:12.706 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: min_mon_release 19 (squid) 2024-04-26T04:48:12.706 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: election_strategy: 1 2024-04-26T04:48:12.706 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-04-26T04:48:12.706 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-04-26T04:48:12.706 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-04-26T04:48:12.706 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:48:12.706 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:48:12.707 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: mgrmap e39: y(active, since 3m), standbys: x 2024-04-26T04:48:12.707 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-26T04:48:12.707 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:48:12.707 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:48:12.707 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: daemon grafana.a on smithi171 is in error state 2024-04-26T04:48:12.707 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:12.707 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:12.707 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:12.707 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:12 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1205962104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:12.713 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: mon.c calling monitor election 2024-04-26T04:48:12.713 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: mon.b calling monitor election 2024-04-26T04:48:12.713 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:12.713 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: last_changed 2024-04-26T04:48:06.508401+0000 2024-04-26T04:48:12.713 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: created 2024-04-26T04:33:43.779990+0000 2024-04-26T04:48:12.713 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: min_mon_release 19 (squid) 2024-04-26T04:48:12.713 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: election_strategy: 1 2024-04-26T04:48:12.713 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-04-26T04:48:12.713 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-04-26T04:48:12.713 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-04-26T04:48:12.713 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:48:12.713 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:48:12.714 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: mgrmap e39: y(active, since 3m), standbys: x 2024-04-26T04:48:12.714 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-26T04:48:12.714 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:48:12.714 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:48:12.714 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: daemon grafana.a on smithi171 is in error state 2024-04-26T04:48:12.714 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: mon.a calling monitor election 2024-04-26T04:48:12.714 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:48:12.714 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: monmap epoch 4 2024-04-26T04:48:12.714 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: fsid 110027f2-0386-11ef-bc93-c7b262605968 2024-04-26T04:48:12.714 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: last_changed 2024-04-26T04:48:06.508401+0000 2024-04-26T04:48:12.714 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: created 2024-04-26T04:33:43.779990+0000 2024-04-26T04:48:12.714 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: min_mon_release 19 (squid) 2024-04-26T04:48:12.714 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: election_strategy: 1 2024-04-26T04:48:12.714 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-04-26T04:48:12.714 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-04-26T04:48:12.715 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-04-26T04:48:12.715 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: fsmap foo:1 {0=foo.smithi156.wyfzii=up:active} 1 up:standby 2024-04-26T04:48:12.715 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:48:12.715 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: mgrmap e39: y(active, since 3m), standbys: x 2024-04-26T04:48:12.715 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-26T04:48:12.715 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:48:12.715 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:48:12.715 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: daemon grafana.a on smithi171 is in error state 2024-04-26T04:48:12.715 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:12.715 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:12.715 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:12.715 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:12 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1205962104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:12.730 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:48:14.795 INFO:teuthology.orchestra.run.smithi156.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:48:14.795 INFO:teuthology.orchestra.run.smithi156.stdout:alertmanager.a smithi156 *:9093,9094 running (4m) 15s ago 8m 24.3M - 0.25.0 c8568f914cd2 932345b7da32 2024-04-26T04:48:14.795 INFO:teuthology.orchestra.run.smithi156.stdout:grafana.a smithi171 *:3000 error 21s ago 8m - - 2024-04-26T04:48:14.795 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi156.wyfzii smithi156 running (7m) 15s ago 7m 16.6M - 17.2.7-902-gc61472c8 0de407c1b3af f42880bb83a3 2024-04-26T04:48:14.796 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi171.kzjlqn smithi171 running (7m) 21s ago 7m 14.4M - 17.2.7-902-gc61472c8 0de407c1b3af 31b518bc187c 2024-04-26T04:48:14.796 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.x smithi171 *:8443,9283,8765 running (3m) 21s ago 12m 450M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 9ec4d348d704 2024-04-26T04:48:14.796 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.y smithi156 *:8443,9283,8765 running (4m) 15s ago 14m 500M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 715bee4dd354 2024-04-26T04:48:14.796 INFO:teuthology.orchestra.run.smithi156.stdout:mon.a smithi156 running (3m) 15s ago 14m 45.5M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b94e4b5539a1 2024-04-26T04:48:14.796 INFO:teuthology.orchestra.run.smithi156.stdout:mon.b smithi171 starting - - - 2048M 2024-04-26T04:48:14.796 INFO:teuthology.orchestra.run.smithi156.stdout:mon.c smithi156 running (104s) 15s ago 12m 31.5M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 8cac6c96e0db 2024-04-26T04:48:14.796 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.a smithi156 *:9100 running (4m) 15s ago 7m 17.5M - 1.5.0 0da6a335fe13 5af6998cc7e3 2024-04-26T04:48:14.796 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.b smithi171 *:9100 running (4m) 21s ago 7m 15.1M - 1.5.0 0da6a335fe13 a2d37cfca8cf 2024-04-26T04:48:14.796 INFO:teuthology.orchestra.run.smithi156.stdout:osd.0 smithi156 running (12m) 15s ago 12m 48.6M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e1273966ada0 2024-04-26T04:48:14.796 INFO:teuthology.orchestra.run.smithi156.stdout:osd.1 smithi156 running (11m) 15s ago 11m 47.8M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 7abca656e994 2024-04-26T04:48:14.796 INFO:teuthology.orchestra.run.smithi156.stdout:osd.2 smithi156 running (11m) 15s ago 11m 53.3M 2480M 17.2.7-902-gc61472c8 0de407c1b3af c951e2171c5d 2024-04-26T04:48:14.796 INFO:teuthology.orchestra.run.smithi156.stdout:osd.3 smithi156 running (11m) 15s ago 11m 45.0M 2480M 17.2.7-902-gc61472c8 0de407c1b3af a2c01ca7770f 2024-04-26T04:48:14.796 INFO:teuthology.orchestra.run.smithi156.stdout:osd.4 smithi171 running (10m) 21s ago 10m 50.6M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 388a24a70001 2024-04-26T04:48:14.796 INFO:teuthology.orchestra.run.smithi156.stdout:osd.5 smithi171 running (10m) 21s ago 10m 53.3M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 703235a355b5 2024-04-26T04:48:14.797 INFO:teuthology.orchestra.run.smithi156.stdout:osd.6 smithi171 running (9m) 21s ago 9m 47.5M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 3dad877f8247 2024-04-26T04:48:14.797 INFO:teuthology.orchestra.run.smithi156.stdout:osd.7 smithi171 running (9m) 21s ago 9m 52.9M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e275b2da9b18 2024-04-26T04:48:14.797 INFO:teuthology.orchestra.run.smithi156.stdout:prometheus.a smithi171 *:9095 running (3m) 21s ago 8m 61.8M - 2.43.0 a07b618ecd1d 93d6b22f6fce 2024-04-26T04:48:14.797 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:14 smithi156 ceph-mon[98743]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:14.797 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:48:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:48:14] "GET /metrics HTTP/1.1" 200 33720 "" "Prometheus/2.43.0" 2024-04-26T04:48:14.798 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:14 smithi156 ceph-mon[103290]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:14.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:14 smithi171 ceph-mon[90566]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:15.453 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T04:48:15.712 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:48:15.834 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:15 smithi156 ceph-mon[98743]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:48:15.834 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:15 smithi156 ceph-mon[103290]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:48:16.002 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:15 smithi171 ceph-mon[90566]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:48:16.902 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:16 smithi156 ceph-mon[98743]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:16.903 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:16 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:16.903 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:16 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:16.903 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:16 smithi156 ceph-mon[103290]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:16.903 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:16 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:16.903 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:16 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:16.939 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:16 smithi171 ceph-mon[90566]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:16.939 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:16 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:16.939 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:16 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:17.659 INFO:teuthology.orchestra.run.smithi156.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:48:17.993 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:17 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:17.993 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:17 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:17.993 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:17 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:17.993 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:17 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:17.994 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:17 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:17.994 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:17 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:17.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:17 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:17.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:17 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:17.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:17 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:18.338 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 60' 2024-04-26T04:48:18.594 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:48:18.671 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:48:18 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:48:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:48:18.951 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:18 smithi171 ceph-mon[90566]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:18.951 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:18 smithi171 ceph-mon[90566]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:48:18.951 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:18 smithi171 ceph-mon[90566]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:48:18.963 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:18 smithi156 ceph-mon[103290]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:18.964 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:18 smithi156 ceph-mon[103290]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:48:18.964 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:18 smithi156 ceph-mon[103290]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:48:18.964 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:18 smithi156 ceph-mon[98743]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:18.964 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:18 smithi156 ceph-mon[98743]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:48:18.964 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:18 smithi156 ceph-mon[98743]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:48:20.954 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[98743]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:20.954 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:20.954 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:20.954 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:20.954 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:20.955 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:20.955 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:20.955 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:48:20.955 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:48:20.955 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:20.955 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[103290]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:20.955 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:20.955 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:20.955 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:20.955 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:20.955 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:20.955 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:20.955 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:48:20.956 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:48:20.956 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:21.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:20 smithi171 ceph-mon[90566]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:21.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:21.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:21.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:21.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:21.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:21.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:21.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:48:21.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:48:21.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:21.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:21 smithi171 ceph-mon[90566]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:21.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:21 smithi171 ceph-mon[90566]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:21.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:21 smithi171 ceph-mon[90566]: Reconfiguring mon.a (monmap changed)... 2024-04-26T04:48:21.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:21 smithi171 ceph-mon[90566]: Reconfiguring daemon mon.a on smithi156 2024-04-26T04:48:22.010 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:21 smithi156 ceph-mon[103290]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:22.010 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:21 smithi156 ceph-mon[103290]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:22.010 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:21 smithi156 ceph-mon[103290]: Reconfiguring mon.a (monmap changed)... 2024-04-26T04:48:22.010 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:21 smithi156 ceph-mon[103290]: Reconfiguring daemon mon.a on smithi156 2024-04-26T04:48:22.011 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:21 smithi156 ceph-mon[98743]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:22.011 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:21 smithi156 ceph-mon[98743]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:22.011 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:21 smithi156 ceph-mon[98743]: Reconfiguring mon.a (monmap changed)... 2024-04-26T04:48:22.011 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:21 smithi156 ceph-mon[98743]: Reconfiguring daemon mon.a on smithi156 2024-04-26T04:48:23.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:23 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:23.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:23 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:23.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:23 smithi156 ceph-mon[103290]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T04:48:23.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:23 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:48:23.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:23 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:48:23.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:23 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:23.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:23 smithi156 ceph-mon[103290]: Reconfiguring daemon mgr.y on smithi156 2024-04-26T04:48:23.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:23 smithi156 ceph-mon[103290]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:23.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:23 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:23.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:23 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:23.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:23 smithi156 ceph-mon[98743]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T04:48:23.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:23 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:48:23.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:23 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:48:23.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:23 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:23.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:23 smithi156 ceph-mon[98743]: Reconfiguring daemon mgr.y on smithi156 2024-04-26T04:48:23.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:23 smithi156 ceph-mon[98743]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:23.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:23 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:23.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:23 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:23.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:23 smithi171 ceph-mon[90566]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T04:48:23.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:23 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:48:23.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:23 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:48:23.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:23 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:23.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:23 smithi171 ceph-mon[90566]: Reconfiguring daemon mgr.y on smithi156 2024-04-26T04:48:23.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:23 smithi171 ceph-mon[90566]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:24.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:24 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:25.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:24 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:25.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:24 smithi171 ceph-mon[90566]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:48:25.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:24 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:48:25.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:24 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:48:25.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:24 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:25.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:24 smithi171 ceph-mon[90566]: Reconfiguring daemon mon.c on smithi156 2024-04-26T04:48:25.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:48:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:48:24] "GET /metrics HTTP/1.1" 200 33787 "" "Prometheus/2.43.0" 2024-04-26T04:48:25.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:24 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:25.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:24 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:25.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:24 smithi156 ceph-mon[98743]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:48:25.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:24 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:48:25.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:24 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:48:25.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:24 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:25.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:24 smithi156 ceph-mon[98743]: Reconfiguring daemon mon.c on smithi156 2024-04-26T04:48:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:24 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:24 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:24 smithi156 ceph-mon[103290]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:48:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:24 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:48:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:24 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:48:25.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:24 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:25.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:24 smithi156 ceph-mon[103290]: Reconfiguring daemon mon.c on smithi156 2024-04-26T04:48:26.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:25 smithi171 ceph-mon[90566]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:26.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:25 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:26.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:25 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:26.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:25 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:48:26.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:25 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:26.064 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:25 smithi156 ceph-mon[98743]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:26.064 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:25 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:26.064 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:25 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:26.064 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:25 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:48:26.064 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:25 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:26.065 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:25 smithi156 ceph-mon[103290]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:26.065 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:25 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:26.065 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:25 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:26.065 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:25 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:48:26.065 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:25 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:27.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:26 smithi156 ceph-mon[98743]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T04:48:27.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:26 smithi156 ceph-mon[98743]: Reconfiguring daemon osd.0 on smithi156 2024-04-26T04:48:27.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:26 smithi156 ceph-mon[103290]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T04:48:27.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:26 smithi156 ceph-mon[103290]: Reconfiguring daemon osd.0 on smithi156 2024-04-26T04:48:27.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:26 smithi171 ceph-mon[90566]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T04:48:27.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:26 smithi171 ceph-mon[90566]: Reconfiguring daemon osd.0 on smithi156 2024-04-26T04:48:28.085 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:27 smithi156 ceph-mon[98743]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:28.085 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:27 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:28.085 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:27 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:28.085 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:27 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:48:28.085 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:27 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:28.086 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:27 smithi156 ceph-mon[103290]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:28.086 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:27 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:28.086 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:27 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:28.086 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:27 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:48:28.086 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:27 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:28.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:27 smithi171 ceph-mon[90566]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:28.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:27 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:28.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:27 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:28.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:27 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:48:28.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:27 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:28.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:28 smithi156 ceph-mon[98743]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T04:48:28.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:28 smithi156 ceph-mon[98743]: Reconfiguring daemon osd.1 on smithi156 2024-04-26T04:48:28.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:28 smithi156 ceph-mon[103290]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T04:48:28.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:28 smithi156 ceph-mon[103290]: Reconfiguring daemon osd.1 on smithi156 2024-04-26T04:48:28.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:28 smithi171 ceph-mon[90566]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T04:48:29.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:28 smithi171 ceph-mon[90566]: Reconfiguring daemon osd.1 on smithi156 2024-04-26T04:48:29.000 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:48:28 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:48:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:48:29.799 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:29 smithi156 ceph-mon[103290]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:29.799 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:29 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:29.799 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:29 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:29.799 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:29 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:48:29.799 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:29 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:29.800 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:29 smithi156 ceph-mon[98743]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:29.800 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:29 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:29.800 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:29 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:29.800 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:29 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:48:29.800 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:29 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:30.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:29 smithi171 ceph-mon[90566]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:30.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:29 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:30.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:29 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:30.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:29 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:48:30.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:29 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:31.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:30 smithi156 ceph-mon[103290]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T04:48:31.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:30 smithi156 ceph-mon[103290]: Reconfiguring daemon osd.2 on smithi156 2024-04-26T04:48:31.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:30 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:31.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:30 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:31.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:30 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:48:31.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:30 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:31.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:30 smithi156 ceph-mon[98743]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T04:48:31.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:30 smithi156 ceph-mon[98743]: Reconfiguring daemon osd.2 on smithi156 2024-04-26T04:48:31.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:30 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:31.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:30 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:31.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:30 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:48:31.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:30 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:31.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:30 smithi171 ceph-mon[90566]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T04:48:31.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:30 smithi171 ceph-mon[90566]: Reconfiguring daemon osd.2 on smithi156 2024-04-26T04:48:31.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:30 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:31.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:30 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:31.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:30 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:48:31.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:30 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:32.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:31 smithi156 ceph-mon[103290]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:32.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:31 smithi156 ceph-mon[103290]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T04:48:32.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:31 smithi156 ceph-mon[103290]: Reconfiguring daemon osd.3 on smithi156 2024-04-26T04:48:32.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:31 smithi156 ceph-mon[98743]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:32.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:31 smithi156 ceph-mon[98743]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T04:48:32.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:31 smithi156 ceph-mon[98743]: Reconfiguring daemon osd.3 on smithi156 2024-04-26T04:48:32.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:31 smithi171 ceph-mon[90566]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:32.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:31 smithi171 ceph-mon[90566]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T04:48:32.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:31 smithi171 ceph-mon[90566]: Reconfiguring daemon osd.3 on smithi156 2024-04-26T04:48:33.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:33 smithi156 ceph-mon[98743]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:33.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:33 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:33.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:33 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:33.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:33 smithi156 ceph-mon[98743]: Reconfiguring mds.foo.smithi156.wyfzii (monmap changed)... 2024-04-26T04:48:33.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:33 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.wyfzii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:48:33.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:33 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:33.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:33 smithi156 ceph-mon[98743]: Reconfiguring daemon mds.foo.smithi156.wyfzii on smithi156 2024-04-26T04:48:33.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:33 smithi156 ceph-mon[103290]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:33.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:33 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:33.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:33 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:33.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:33 smithi156 ceph-mon[103290]: Reconfiguring mds.foo.smithi156.wyfzii (monmap changed)... 2024-04-26T04:48:33.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:33 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.wyfzii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:48:33.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:33 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:33.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:33 smithi156 ceph-mon[103290]: Reconfiguring daemon mds.foo.smithi156.wyfzii on smithi156 2024-04-26T04:48:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:33 smithi171 ceph-mon[90566]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:33 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:33 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:33 smithi171 ceph-mon[90566]: Reconfiguring mds.foo.smithi156.wyfzii (monmap changed)... 2024-04-26T04:48:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:33 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.wyfzii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:48:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:33 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:33.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:33 smithi171 ceph-mon[90566]: Reconfiguring daemon mds.foo.smithi156.wyfzii on smithi156 2024-04-26T04:48:35.063 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:48:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:48:34] "GET /metrics HTTP/1.1" 200 33787 "" "Prometheus/2.43.0" 2024-04-26T04:48:35.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:35 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:35.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:35 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:35.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:35 smithi171 ceph-mon[90566]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:48:35.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:35 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:48:35.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:35 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:48:35.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:35 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:35.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:35 smithi171 ceph-mon[90566]: Reconfiguring daemon mon.b on smithi171 2024-04-26T04:48:35.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:35 smithi171 ceph-mon[90566]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:35.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:35 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:35.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:35.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:35.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[103290]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:48:35.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:48:35.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:48:35.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:35.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[103290]: Reconfiguring daemon mon.b on smithi171 2024-04-26T04:48:35.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[103290]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:35.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:35.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:35.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:35.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[98743]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:48:35.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:48:35.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:48:35.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:35.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[98743]: Reconfiguring daemon mon.b on smithi171 2024-04-26T04:48:35.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[98743]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:35.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:36.798 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:36 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:36.798 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:36 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:36.799 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:36 smithi171 ceph-mon[90566]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T04:48:36.799 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:36 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:48:36.799 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:36 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:48:36.799 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:36 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:36.799 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:36 smithi171 ceph-mon[90566]: Reconfiguring daemon mgr.x on smithi171 2024-04-26T04:48:37.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:36 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:37.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:36 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:37.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:36 smithi156 ceph-mon[98743]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T04:48:37.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:36 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:48:37.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:36 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:48:37.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:36 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:37.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:36 smithi156 ceph-mon[98743]: Reconfiguring daemon mgr.x on smithi171 2024-04-26T04:48:37.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:36 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:37.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:36 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:37.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:36 smithi156 ceph-mon[103290]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T04:48:37.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:36 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:48:37.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:36 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:48:37.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:36 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:37.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:36 smithi156 ceph-mon[103290]: Reconfiguring daemon mgr.x on smithi171 2024-04-26T04:48:38.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:37 smithi156 ceph-mon[103290]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:38.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:37 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:38.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:37 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:38.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:37 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:48:38.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:37 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:38.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:37 smithi156 ceph-mon[98743]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:38.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:37 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:38.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:37 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:38.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:37 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:48:38.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:37 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:38.096 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:37 smithi171 ceph-mon[90566]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:38.096 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:37 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:38.096 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:37 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:38.096 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:37 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:48:38.096 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:37 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:38.749 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:48:38 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:48:38.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi171.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:48:39.075 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:38 smithi171 ceph-mon[90566]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T04:48:39.075 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:38 smithi171 ceph-mon[90566]: Reconfiguring daemon osd.4 on smithi171 2024-04-26T04:48:39.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:38 smithi156 ceph-mon[98743]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T04:48:39.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:38 smithi156 ceph-mon[98743]: Reconfiguring daemon osd.4 on smithi171 2024-04-26T04:48:39.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:38 smithi156 ceph-mon[103290]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T04:48:39.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:38 smithi156 ceph-mon[103290]: Reconfiguring daemon osd.4 on smithi171 2024-04-26T04:48:40.053 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:39 smithi171 ceph-mon[90566]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:40.053 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:39 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:40.053 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:39 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:40.053 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:39 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:48:40.053 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:39 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:40.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:39 smithi156 ceph-mon[103290]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:40.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:39 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:40.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:39 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:40.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:39 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:48:40.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:39 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:40.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:39 smithi156 ceph-mon[98743]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:40.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:39 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:40.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:39 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:40.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:39 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:48:40.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:39 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:41.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:40 smithi156 ceph-mon[98743]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T04:48:41.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:40 smithi156 ceph-mon[98743]: Reconfiguring daemon osd.5 on smithi171 2024-04-26T04:48:41.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:40 smithi156 ceph-mon[103290]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T04:48:41.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:40 smithi156 ceph-mon[103290]: Reconfiguring daemon osd.5 on smithi171 2024-04-26T04:48:41.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:40 smithi171 ceph-mon[90566]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T04:48:41.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:40 smithi171 ceph-mon[90566]: Reconfiguring daemon osd.5 on smithi171 2024-04-26T04:48:42.211 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:41 smithi156 ceph-mon[103290]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:42.211 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:41 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:42.212 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:41 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:42.212 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:41 smithi156 ceph-mon[103290]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T04:48:42.212 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:41 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:48:42.212 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:41 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:42.212 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:41 smithi156 ceph-mon[103290]: Reconfiguring daemon osd.6 on smithi171 2024-04-26T04:48:42.212 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:41 smithi156 ceph-mon[98743]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:42.212 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:41 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:42.212 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:41 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:42.212 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:41 smithi156 ceph-mon[98743]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T04:48:42.212 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:41 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:48:42.212 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:41 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:42.213 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:41 smithi156 ceph-mon[98743]: Reconfiguring daemon osd.6 on smithi171 2024-04-26T04:48:42.230 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:41 smithi171 ceph-mon[90566]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:42.231 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:41 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:42.231 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:41 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:42.231 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:41 smithi171 ceph-mon[90566]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T04:48:42.231 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:41 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:48:42.231 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:41 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:42.231 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:41 smithi171 ceph-mon[90566]: Reconfiguring daemon osd.6 on smithi171 2024-04-26T04:48:43.962 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:43 smithi171 ceph-mon[90566]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:43.962 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:43 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:43.962 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:43 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:43.963 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:43 smithi171 ceph-mon[90566]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T04:48:43.963 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:43 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:48:43.963 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:43 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:43.963 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:43 smithi171 ceph-mon[90566]: Reconfiguring daemon osd.7 on smithi171 2024-04-26T04:48:44.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:43 smithi156 ceph-mon[98743]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:44.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:43 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:44.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:43 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:44.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:43 smithi156 ceph-mon[98743]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T04:48:44.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:43 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:48:44.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:43 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:44.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:43 smithi156 ceph-mon[98743]: Reconfiguring daemon osd.7 on smithi171 2024-04-26T04:48:44.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:43 smithi156 ceph-mon[103290]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:44.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:43 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:44.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:43 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:44.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:43 smithi156 ceph-mon[103290]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T04:48:44.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:43 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:48:44.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:43 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:44.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:43 smithi156 ceph-mon[103290]: Reconfiguring daemon osd.7 on smithi171 2024-04-26T04:48:45.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:48:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:48:44] "GET /metrics HTTP/1.1" 200 33785 "" "Prometheus/2.43.0" 2024-04-26T04:48:45.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:45 smithi171 ceph-mon[90566]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:45.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:45 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:45.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:45 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:45.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:45 smithi171 ceph-mon[90566]: Reconfiguring mds.foo.smithi171.kzjlqn (monmap changed)... 2024-04-26T04:48:45.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:45 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.kzjlqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:48:45.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:45 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:45.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:45 smithi171 ceph-mon[90566]: Reconfiguring daemon mds.foo.smithi171.kzjlqn on smithi171 2024-04-26T04:48:45.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:45 smithi156 ceph-mon[98743]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:45.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:45 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:45.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:45 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:45.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:45 smithi156 ceph-mon[98743]: Reconfiguring mds.foo.smithi171.kzjlqn (monmap changed)... 2024-04-26T04:48:45.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:45 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.kzjlqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:48:45.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:45 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:45.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:45 smithi156 ceph-mon[98743]: Reconfiguring daemon mds.foo.smithi171.kzjlqn on smithi171 2024-04-26T04:48:45.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:45 smithi156 ceph-mon[103290]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:45.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:45 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:45.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:45 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:45.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:45 smithi156 ceph-mon[103290]: Reconfiguring mds.foo.smithi171.kzjlqn (monmap changed)... 2024-04-26T04:48:45.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:45 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.kzjlqn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:48:45.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:45 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:45.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:45 smithi156 ceph-mon[103290]: Reconfiguring daemon mds.foo.smithi171.kzjlqn on smithi171 2024-04-26T04:48:47.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:47 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:47.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:47 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:47.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:47 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:47.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:47 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:47.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:47 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:47.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:47 smithi171 ceph-mon[90566]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:47.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:47 smithi171 ceph-mon[90566]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:47.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:47 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:47.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:47 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:47.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:47 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:47.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:47 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:47.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:47 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:47.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:47 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:47.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:47 smithi156 ceph-mon[98743]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:47.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:47 smithi156 ceph-mon[98743]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:47.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:47 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:47.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:47 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:47.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:47 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:47.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:47 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:47.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:47 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:47.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:47 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:47.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:47 smithi156 ceph-mon[103290]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:47.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:47 smithi156 ceph-mon[103290]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:47.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:47 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:48:49.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:49 smithi171 ceph-mon[90566]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:49.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:49 smithi156 ceph-mon[98743]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:49.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:49 smithi156 ceph-mon[103290]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:50.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:50 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:50.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:50 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:50.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:50 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:51.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:51 smithi171 ceph-mon[90566]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:51.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:51 smithi156 ceph-mon[98743]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:51.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:51 smithi156 ceph-mon[103290]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:53.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:53 smithi171 ceph-mon[90566]: pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:53.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:53 smithi156 ceph-mon[98743]: pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:53.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:53 smithi156 ceph-mon[103290]: pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:55.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:48:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:48:54] "GET /metrics HTTP/1.1" 200 33785 "" "Prometheus/2.43.0" 2024-04-26T04:48:55.567 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:55 smithi171 ceph-mon[90566]: pgmap v128: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:55.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:55 smithi156 ceph-mon[98743]: pgmap v128: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:55.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:55 smithi156 ceph-mon[103290]: pgmap v128: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:57.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:57 smithi156 ceph-mon[98743]: pgmap v129: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:57.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:57 smithi156 ceph-mon[103290]: pgmap v129: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:57.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:57 smithi171 ceph-mon[90566]: pgmap v129: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:59.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:48:59 smithi156 ceph-mon[98743]: pgmap v130: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:59.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:48:59 smithi156 ceph-mon[103290]: pgmap v130: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:48:59.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:48:59 smithi171 ceph-mon[90566]: pgmap v130: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:01.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:01 smithi156 ceph-mon[98743]: pgmap v131: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:01.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:01 smithi156 ceph-mon[103290]: pgmap v131: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:01.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:01 smithi171 ceph-mon[90566]: pgmap v131: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:03.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:03 smithi156 ceph-mon[103290]: pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:03.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:03 smithi156 ceph-mon[98743]: pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:03.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:03 smithi171 ceph-mon[90566]: pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:05.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:49:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:49:04] "GET /metrics HTTP/1.1" 200 33785 "" "Prometheus/2.43.0" 2024-04-26T04:49:05.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:05 smithi156 ceph-mon[98743]: pgmap v133: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:05.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:05.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:05 smithi156 ceph-mon[103290]: pgmap v133: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:05.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:05 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:05.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:05 smithi171 ceph-mon[90566]: pgmap v133: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:05.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:05 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:07.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:07 smithi156 ceph-mon[98743]: pgmap v134: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:07.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:07 smithi156 ceph-mon[103290]: pgmap v134: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:07.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:07 smithi171 ceph-mon[90566]: pgmap v134: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:09.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:09 smithi156 ceph-mon[98743]: pgmap v135: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:09.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:09 smithi156 ceph-mon[103290]: pgmap v135: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:09.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:09 smithi171 ceph-mon[90566]: pgmap v135: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:11.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:11 smithi156 ceph-mon[98743]: pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:11.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:11 smithi156 ceph-mon[103290]: pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:11.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:11 smithi171 ceph-mon[90566]: pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:13.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:13 smithi156 ceph-mon[98743]: pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:13.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:13 smithi156 ceph-mon[103290]: pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:13.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:13 smithi171 ceph-mon[90566]: pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:15.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:49:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:49:14] "GET /metrics HTTP/1.1" 200 33785 "" "Prometheus/2.43.0" 2024-04-26T04:49:15.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:15 smithi156 ceph-mon[98743]: pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:15.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:15 smithi156 ceph-mon[103290]: pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:15.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:15 smithi171 ceph-mon[90566]: pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:17.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:17 smithi156 ceph-mon[98743]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:17.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:17 smithi156 ceph-mon[103290]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:17.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:17 smithi171 ceph-mon[90566]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:19.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:19 smithi156 ceph-mon[103290]: pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:19.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:19 smithi156 ceph-mon[98743]: pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:19.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:19 smithi171 ceph-mon[90566]: pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:20.528 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:20.529 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:20.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:20.874 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade resume' 2024-04-26T04:49:21.138 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:21.556 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:21 smithi156 ceph-mon[98743]: pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:21.556 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:21 smithi156 ceph-mon[103290]: pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:21.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:21 smithi171 ceph-mon[90566]: pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:23.050 INFO:teuthology.orchestra.run.smithi156.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:23.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:23 smithi156 ceph-mon[103290]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:23.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:23 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:23.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:23 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:23.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:23 smithi156 ceph-mon[98743]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:23.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:23 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:23.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:23 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:23.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:23 smithi171 ceph-mon[90566]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:23.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:23 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:23.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:23 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:23.774 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for half of osds to upgrade' 2024-04-26T04:49:24.408 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:24.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:24 smithi156 ceph-mon[98743]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:24.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:24 smithi156 ceph-mon[98743]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:24.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:24 smithi156 ceph-mon[103290]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:24.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:24 smithi156 ceph-mon[103290]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:24.740 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:24 smithi171 ceph-mon[90566]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:24.740 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:24 smithi171 ceph-mon[90566]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:24.996 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:49:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:49:24] "GET /metrics HTTP/1.1" 200 33718 "" "Prometheus/2.43.0" 2024-04-26T04:49:25.576 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:25 smithi171 ceph-mon[90566]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:25.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:25 smithi156 ceph-mon[98743]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:25.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:25 smithi156 ceph-mon[103290]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:27.161 INFO:teuthology.orchestra.run.smithi156.stdout:wait for half of osds to upgrade 2024-04-26T04:49:27.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:27 smithi156 ceph-mon[98743]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:27.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:27 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:27.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:27 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:27.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:27 smithi156 ceph-mon[103290]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:27.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:27 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:27.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:27 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:27.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:27 smithi171 ceph-mon[90566]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:27.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:27 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:27.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:27 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:27.949 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-04-26T04:49:28.708 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:29.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:29 smithi156 ceph-mon[98743]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:29.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:29 smithi156 ceph-mon[103290]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:29.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:29 smithi171 ceph-mon[90566]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:31.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:31 smithi156 ceph-mon[103290]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:31.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:31 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:31.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:31 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:31.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:31 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2730566348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:31.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:31 smithi156 ceph-mon[98743]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:31.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:31 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:31.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:31 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:31.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:31 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2730566348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:31.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:31 smithi171 ceph-mon[90566]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:31.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:31 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:31.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:31 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:31.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:31 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2730566348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:32.557 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:32.557 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:32.557 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:32.557 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:32.557 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:32.557 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:32.557 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:32.557 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:32.557 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: Upgrade: Setting container_image for all mon 2024-04-26T04:49:32.557 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:32.557 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T04:49:32.558 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T04:49:32.558 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T04:49:32.558 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T04:49:32.558 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T04:49:32.558 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T04:49:32.558 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:32.558 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: Upgrade: Setting container_image for all crash 2024-04-26T04:49:32.558 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:32.558 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:49:32.558 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:49:32.558 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[98743]: Upgrade: osd.0 is safe to restart 2024-04-26T04:49:32.558 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:32.559 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:32.559 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:32.559 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:32.559 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:32.559 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:32.559 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:32.559 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:32.559 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: Upgrade: Setting container_image for all mon 2024-04-26T04:49:32.559 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:32.559 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T04:49:32.559 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T04:49:32.559 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T04:49:32.559 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T04:49:32.560 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T04:49:32.560 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T04:49:32.560 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:32.560 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: Upgrade: Setting container_image for all crash 2024-04-26T04:49:32.560 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:32.560 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:49:32.560 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:49:32.560 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:32 smithi156 ceph-mon[103290]: Upgrade: osd.0 is safe to restart 2024-04-26T04:49:32.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:32.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:32.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:32.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:32.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:32.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:32.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:32.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:32.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: Upgrade: Setting container_image for all mon 2024-04-26T04:49:32.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:32.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T04:49:32.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T04:49:32.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T04:49:32.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T04:49:32.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T04:49:32.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T04:49:32.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:32.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: Upgrade: Setting container_image for all crash 2024-04-26T04:49:32.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:32.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:49:32.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:49:32.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:32 smithi171 ceph-mon[90566]: Upgrade: osd.0 is safe to restart 2024-04-26T04:49:34.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:34 smithi156 ceph-mon[98743]: Upgrade: Updating osd.0 2024-04-26T04:49:34.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:34 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:34.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:34 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:49:34.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:34 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:34.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:34 smithi156 ceph-mon[98743]: Deploying daemon osd.0 on smithi156 2024-04-26T04:49:34.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:34 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2432809451' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:34.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:34 smithi156 ceph-mon[98743]: pgmap v149: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:34.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:34 smithi156 ceph-mon[103290]: Upgrade: Updating osd.0 2024-04-26T04:49:34.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:34 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:34.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:34 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:49:34.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:34 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:34.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:34 smithi156 ceph-mon[103290]: Deploying daemon osd.0 on smithi156 2024-04-26T04:49:34.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:34 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2432809451' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:34.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:34 smithi156 ceph-mon[103290]: pgmap v149: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:34.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:34 smithi171 ceph-mon[90566]: Upgrade: Updating osd.0 2024-04-26T04:49:34.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:34 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:34.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:34 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:49:34.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:34 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:34.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:34 smithi171 ceph-mon[90566]: Deploying daemon osd.0 on smithi156 2024-04-26T04:49:34.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:34 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2432809451' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:34.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:34 smithi171 ceph-mon[90566]: pgmap v149: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:34.901 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:49:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:49:34] "GET /metrics HTTP/1.1" 200 33718 "" "Prometheus/2.43.0" 2024-04-26T04:49:35.579 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:35.580 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:35.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:35 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:36.565 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:36 smithi156 ceph-mon[98743]: pgmap v150: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:36.565 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:36 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/231740830' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:36.565 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:36 smithi156 ceph-mon[103290]: pgmap v150: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:36.566 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:36 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/231740830' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:36.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:36 smithi171 ceph-mon[90566]: pgmap v150: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:36.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:36 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/231740830' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:36.843 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:36 smithi156 systemd[1]: Stopping Ceph osd.0 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:49:36.843 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:36 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0[43431]: 2024-04-26T04:49:36.806+0000 7fcdad5b7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:49:36.843 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:36 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0[43431]: 2024-04-26T04:49:36.806+0000 7fcdad5b7700 -1 osd.0 78 *** Got signal Terminated *** 2024-04-26T04:49:36.844 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:36 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0[43431]: 2024-04-26T04:49:36.806+0000 7fcdad5b7700 -1 osd.0 78 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:49:37.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:37 smithi156 ceph-mon[98743]: osd.0 marked itself down and dead 2024-04-26T04:49:37.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:37 smithi156 ceph-mon[103290]: osd.0 marked itself down and dead 2024-04-26T04:49:37.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:37 smithi171 ceph-mon[90566]: osd.0 marked itself down and dead 2024-04-26T04:49:38.014 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:37 smithi156 podman[112342]: 2024-04-26 04:49:37.726739671 +0000 UTC m=+1.019386640 container died e1273966ada0c94824a7f05da0396c027e9a4f35d986ddf961afc88258ebc4bf (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-26T04:49:38.315 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:38 smithi156 podman[112342]: 2024-04-26 04:49:38.08697648 +0000 UTC m=+1.379623444 container cleanup e1273966ada0c94824a7f05da0396c027e9a4f35d986ddf961afc88258ebc4bf (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, ceph=True) 2024-04-26T04:49:38.315 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:38 smithi156 bash[112342]: ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0 2024-04-26T04:49:38.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:38 smithi156 ceph-mon[98743]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:49:38.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:38 smithi156 ceph-mon[98743]: osdmap e79: 8 total, 7 up, 8 in 2024-04-26T04:49:38.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:38 smithi156 ceph-mon[98743]: pgmap v152: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:38.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:38 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2220772699' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:38.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:38 smithi156 ceph-mon[103290]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:49:38.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:38 smithi156 ceph-mon[103290]: osdmap e79: 8 total, 7 up, 8 in 2024-04-26T04:49:38.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:38 smithi156 ceph-mon[103290]: pgmap v152: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:38.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:38 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2220772699' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:38.594 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:38 smithi156 podman[112364]: 2024-04-26 04:49:38.338415923 +0000 UTC m=+0.612408499 container remove e1273966ada0c94824a7f05da0396c027e9a4f35d986ddf961afc88258ebc4bf (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0, ceph=True, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-26T04:49:38.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:38 smithi171 ceph-mon[90566]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:49:38.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:38 smithi171 ceph-mon[90566]: osdmap e79: 8 total, 7 up, 8 in 2024-04-26T04:49:38.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:38 smithi171 ceph-mon[90566]: pgmap v152: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:38.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:38 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2220772699' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:38.959 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:38 smithi156 podman[112485]: 2024-04-26 04:49:38.557717638 +0000 UTC m=+0.018193759 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:38.959 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:38 smithi156 podman[112485]: 2024-04-26 04:49:38.681265825 +0000 UTC m=+0.141741930 container create 13e0e39b8143f0ce16045806a4e954e2a6715b8370f67305077c30ab266c6647 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-deactivate, RELEASE=squid-b22e2eb, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-26T04:49:39.264 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:38 smithi156 podman[112485]: 2024-04-26 04:49:38.959175242 +0000 UTC m=+0.419651349 container init 13e0e39b8143f0ce16045806a4e954e2a6715b8370f67305077c30ab266c6647 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-26T04:49:39.264 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:38 smithi156 podman[112485]: 2024-04-26 04:49:38.96535137 +0000 UTC m=+0.425827478 container start 13e0e39b8143f0ce16045806a4e954e2a6715b8370f67305077c30ab266c6647 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-26T04:49:39.264 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:39 smithi156 podman[112485]: 2024-04-26 04:49:39.067582482 +0000 UTC m=+0.528058591 container attach 13e0e39b8143f0ce16045806a4e954e2a6715b8370f67305077c30ab266c6647 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD) 2024-04-26T04:49:39.562 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:39 smithi156 ceph-mon[103290]: osdmap e80: 8 total, 7 up, 8 in 2024-04-26T04:49:39.563 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:39 smithi156 ceph-mon[98743]: osdmap e80: 8 total, 7 up, 8 in 2024-04-26T04:49:39.563 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:39 smithi156 podman[112485]: 2024-04-26 04:49:39.264195834 +0000 UTC m=+0.724671941 container died 13e0e39b8143f0ce16045806a4e954e2a6715b8370f67305077c30ab266c6647 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:49:39.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:39 smithi171 ceph-mon[90566]: osdmap e80: 8 total, 7 up, 8 in 2024-04-26T04:49:40.093 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:39 smithi156 podman[112531]: 2024-04-26 04:49:39.829302571 +0000 UTC m=+0.559980768 container remove 13e0e39b8143f0ce16045806a4e954e2a6715b8370f67305077c30ab266c6647 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-26T04:49:40.093 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:39 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-26T04:49:40.093 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:39 smithi156 systemd[1]: Stopped Ceph osd.0 for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:40.094 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:39 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.0.service: Consumed 5.685s CPU time. 2024-04-26T04:49:40.094 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 systemd[1]: Starting Ceph osd.0 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:49:40.517 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:40 smithi156 ceph-mon[98743]: pgmap v154: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:40.518 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:40 smithi156 ceph-mon[103290]: pgmap v154: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:40.518 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 podman[112639]: 2024-04-26 04:49:40.171317813 +0000 UTC m=+0.027702888 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:40.518 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 podman[112639]: 2024-04-26 04:49:40.288961799 +0000 UTC m=+0.145346851 container create 09383d9929d7cd70cb3e891c5bdabd0837048d932fe0f32d85eba4c230eee661 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS) 2024-04-26T04:49:40.518 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 podman[112639]: 2024-04-26 04:49:40.516944282 +0000 UTC m=+0.373329343 container init 09383d9929d7cd70cb3e891c5bdabd0837048d932fe0f32d85eba4c230eee661 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-26T04:49:40.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:40 smithi171 ceph-mon[90566]: pgmap v154: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:40.844 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 podman[112639]: 2024-04-26 04:49:40.523053443 +0000 UTC m=+0.379438500 container start 09383d9929d7cd70cb3e891c5bdabd0837048d932fe0f32d85eba4c230eee661 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-26T04:49:40.844 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 podman[112639]: 2024-04-26 04:49:40.608779271 +0000 UTC m=+0.465164337 container attach 09383d9929d7cd70cb3e891c5bdabd0837048d932fe0f32d85eba4c230eee661 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:49:41.332 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-activate[112677]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T04:49:41.332 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-activate[112677]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T04:49:41.332 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 bash[112639]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T04:49:41.332 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 bash[112639]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T04:49:41.333 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 bash[112639]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T04:49:41.333 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 bash[112639]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T04:49:41.333 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 bash[112639]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-26T04:49:41.333 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 bash[112639]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T04:49:41.333 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 bash[112639]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-26T04:49:41.334 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-activate[112677]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T04:49:41.334 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-activate[112677]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T04:49:41.334 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-activate[112677]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-26T04:49:41.334 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-activate[112677]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T04:49:41.334 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-activate[112677]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-26T04:49:41.334 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:40 smithi156 podman[112639]: 2024-04-26 04:49:40.991121898 +0000 UTC m=+0.847506961 container died 09383d9929d7cd70cb3e891c5bdabd0837048d932fe0f32d85eba4c230eee661 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-26T04:49:41.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:41 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/682845075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:41.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:41 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/682845075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:41.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:41 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/682845075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:42.093 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:41 smithi156 podman[112818]: 2024-04-26 04:49:41.713068357 +0000 UTC m=+0.716952882 container remove 09383d9929d7cd70cb3e891c5bdabd0837048d932fe0f32d85eba4c230eee661 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:49:42.093 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:41 smithi156 podman[112890]: 2024-04-26 04:49:41.839254486 +0000 UTC m=+0.018464175 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:42.093 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:41 smithi156 podman[112890]: 2024-04-26 04:49:41.955474225 +0000 UTC m=+0.134683897 container create 7d9e99b5875cced995406151ea450ff3594670a4a1bf388241eda514afa16d79 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-26T04:49:42.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:42 smithi156 ceph-mon[98743]: pgmap v155: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:49:42.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:42 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:42.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:42 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:42.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:42 smithi156 ceph-mon[103290]: pgmap v155: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:49:42.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:42 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:42.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:42 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:42.594 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:42 smithi156 podman[112890]: 2024-04-26 04:49:42.166086931 +0000 UTC m=+0.345296604 container init 7d9e99b5875cced995406151ea450ff3594670a4a1bf388241eda514afa16d79 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS) 2024-04-26T04:49:42.595 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:42 smithi156 podman[112890]: 2024-04-26 04:49:42.171558369 +0000 UTC m=+0.350768050 container start 7d9e99b5875cced995406151ea450ff3594670a4a1bf388241eda514afa16d79 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS) 2024-04-26T04:49:42.595 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:42 smithi156 ceph-osd[112928]: -- 172.21.15.156:0/2857862860 <== mon.2 v2:172.21.15.156:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f38f8c9680 con 0x55f38f892c00 2024-04-26T04:49:42.595 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:42 smithi156 bash[112890]: 7d9e99b5875cced995406151ea450ff3594670a4a1bf388241eda514afa16d79 2024-04-26T04:49:42.595 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:42 smithi156 systemd[1]: Started Ceph osd.0 for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:49:42.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:42 smithi171 ceph-mon[90566]: pgmap v155: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:49:42.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:42 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:42.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:42 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:43.093 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:42 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0[112905]: 2024-04-26T04:49:42.713+0000 7f8598ee97c0 -1 Falling back to public interface 2024-04-26T04:49:43.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:43 smithi156 ceph-mon[98743]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:49:43.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:43 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:43.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:43 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1499287853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:43.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:43 smithi156 ceph-mon[103290]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:49:43.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:43 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:43.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:43 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1499287853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:43.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:43 smithi171 ceph-mon[90566]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:49:43.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:43 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:43.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:43 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1499287853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:44.667 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:44 smithi156 ceph-mon[98743]: pgmap v156: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:49:44.667 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:44 smithi156 ceph-mon[103290]: pgmap v156: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:49:44.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:44 smithi171 ceph-mon[90566]: pgmap v156: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:49:45.020 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:49:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:49:44] "GET /metrics HTTP/1.1" 200 33782 "" "Prometheus/2.43.0" 2024-04-26T04:49:45.631 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:45 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1530380458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:45.631 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:45 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1530380458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:45.631 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:45 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0[112905]: 2024-04-26T04:49:45.450+0000 7f8598ee97c0 -1 osd.0 78 log_to_monitors true 2024-04-26T04:49:45.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:45 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1530380458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:46.617 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:46 smithi156 ceph-mon[98743]: from='osd.0 [v2:172.21.15.156:6802/1734867425,v1:172.21.15.156:6803/1734867425]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:49:46.617 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:46 smithi156 ceph-mon[98743]: pgmap v157: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:49:46.617 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:46 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:46.618 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:46 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:46.618 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:46 smithi156 ceph-mon[103290]: from='osd.0 [v2:172.21.15.156:6802/1734867425,v1:172.21.15.156:6803/1734867425]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:49:46.618 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:46 smithi156 ceph-mon[103290]: pgmap v157: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:49:46.618 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:46 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:46.618 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:46 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:46.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:46 smithi171 ceph-mon[90566]: from='osd.0 [v2:172.21.15.156:6802/1734867425,v1:172.21.15.156:6803/1734867425]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:49:46.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:46 smithi171 ceph-mon[90566]: pgmap v157: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:49:46.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:46 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:46.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:46 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:47.093 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:49:46 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0[112905]: 2024-04-26T04:49:46.615+0000 7f8586339700 -1 osd.0 78 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:49:47.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:47 smithi171 ceph-mon[90566]: from='osd.0 [v2:172.21.15.156:6802/1734867425,v1:172.21.15.156:6803/1734867425]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:49:47.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:47 smithi171 ceph-mon[90566]: osdmap e81: 8 total, 7 up, 8 in 2024-04-26T04:49:47.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:47 smithi171 ceph-mon[90566]: from='osd.0 [v2:172.21.15.156:6802/1734867425,v1:172.21.15.156:6803/1734867425]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:49:47.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:47 smithi171 ceph-mon[90566]: from='osd.0 [v2:172.21.15.156:6802/1734867425,v1:172.21.15.156:6803/1734867425]' entity='osd.0' 2024-04-26T04:49:47.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:47 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:47.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:47 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:47.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:47 smithi156 ceph-mon[103290]: from='osd.0 [v2:172.21.15.156:6802/1734867425,v1:172.21.15.156:6803/1734867425]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:49:47.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:47 smithi156 ceph-mon[103290]: osdmap e81: 8 total, 7 up, 8 in 2024-04-26T04:49:47.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:47 smithi156 ceph-mon[103290]: from='osd.0 [v2:172.21.15.156:6802/1734867425,v1:172.21.15.156:6803/1734867425]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:49:47.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:47 smithi156 ceph-mon[103290]: from='osd.0 [v2:172.21.15.156:6802/1734867425,v1:172.21.15.156:6803/1734867425]' entity='osd.0' 2024-04-26T04:49:47.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:47 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:47.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:47 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:47.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:47 smithi156 ceph-mon[98743]: from='osd.0 [v2:172.21.15.156:6802/1734867425,v1:172.21.15.156:6803/1734867425]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:49:47.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:47 smithi156 ceph-mon[98743]: osdmap e81: 8 total, 7 up, 8 in 2024-04-26T04:49:47.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:47 smithi156 ceph-mon[98743]: from='osd.0 [v2:172.21.15.156:6802/1734867425,v1:172.21.15.156:6803/1734867425]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:49:47.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:47 smithi156 ceph-mon[98743]: from='osd.0 [v2:172.21.15.156:6802/1734867425,v1:172.21.15.156:6803/1734867425]' entity='osd.0' 2024-04-26T04:49:47.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:47 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:47.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:47 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:48.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:48 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/45508047' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:48.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:48 smithi156 ceph-mon[103290]: pgmap v159: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:49:48.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:48 smithi156 ceph-mon[103290]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:49:48.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:48 smithi156 ceph-mon[103290]: osd.0 [v2:172.21.15.156:6802/1734867425,v1:172.21.15.156:6803/1734867425] boot 2024-04-26T04:49:48.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:48 smithi156 ceph-mon[103290]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:49:48.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:48 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:49:48.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:48 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/45508047' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:48.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:48 smithi156 ceph-mon[98743]: pgmap v159: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:49:48.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:48 smithi156 ceph-mon[98743]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:49:48.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:48 smithi156 ceph-mon[98743]: osd.0 [v2:172.21.15.156:6802/1734867425,v1:172.21.15.156:6803/1734867425] boot 2024-04-26T04:49:48.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:48 smithi156 ceph-mon[98743]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:49:48.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:48 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:49:48.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:48 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/45508047' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:48 smithi171 ceph-mon[90566]: pgmap v159: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:48 smithi171 ceph-mon[90566]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:48 smithi171 ceph-mon[90566]: osd.0 [v2:172.21.15.156:6802/1734867425,v1:172.21.15.156:6803/1734867425] boot 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:48 smithi171 ceph-mon[90566]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:48 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:49:49.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:49 smithi171 ceph-mon[90566]: osdmap e83: 8 total, 8 up, 8 in 2024-04-26T04:49:50.019 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:49 smithi156 ceph-mon[103290]: osdmap e83: 8 total, 8 up, 8 in 2024-04-26T04:49:50.020 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:49 smithi156 ceph-mon[98743]: osdmap e83: 8 total, 8 up, 8 in 2024-04-26T04:49:50.966 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:50 smithi156 ceph-mon[98743]: pgmap v162: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:49:50.966 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:50 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/4221818591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:50.966 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:50 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:50.966 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:50 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:50.967 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:50 smithi156 ceph-mon[103290]: pgmap v162: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:49:50.967 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:50 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/4221818591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:50.967 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:50 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:50.967 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:50 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:50.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:50 smithi171 ceph-mon[90566]: pgmap v162: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T04:49:51.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:50 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/4221818591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:51.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:50 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:51.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:50 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:51.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:49:51 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:49:51.190+0000 7f741c426700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:49:52.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:52.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:52.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:52.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[98743]: pgmap v163: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:49:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[98743]: pgmap v164: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:49:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:49:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[98743]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:49:52.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[98743]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:49:52.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:52.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[103290]: pgmap v163: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:49:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[103290]: pgmap v164: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:49:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:49:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[103290]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:49:52.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:52 smithi156 ceph-mon[103290]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:49:52.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:52 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:52.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:52 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:52.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:52 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:52.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:52 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:52.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:52 smithi171 ceph-mon[90566]: pgmap v163: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:49:52.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:52 smithi171 ceph-mon[90566]: pgmap v164: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:49:52.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:52 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:49:52.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:52 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:52.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:52 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:52.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:52 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:52.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:52 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:52.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:52 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:49:52.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:52 smithi171 ceph-mon[90566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:49:52.501 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:52 smithi171 ceph-mon[90566]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:49:53.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:53 smithi156 ceph-mon[103290]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:49:53.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:53 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1021617731' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:53.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:53 smithi156 ceph-mon[98743]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:49:53.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:53 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1021617731' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:53.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:53 smithi171 ceph-mon[90566]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:49:53.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:53 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1021617731' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:54.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:54 smithi156 ceph-mon[103290]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:54.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:54 smithi156 ceph-mon[98743]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:54.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:54 smithi171 ceph-mon[90566]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:55.044 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:49:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:49:54] "GET /metrics HTTP/1.1" 200 33698 "" "Prometheus/2.43.0" 2024-04-26T04:49:55.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:55 smithi156 ceph-mon[98743]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2024-04-26T04:49:55.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:55 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/4218968544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:55.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:55 smithi156 ceph-mon[103290]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2024-04-26T04:49:55.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:55 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/4218968544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:55.401 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:55 smithi171 ceph-mon[90566]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2024-04-26T04:49:55.401 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:55 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/4218968544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:56.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:56 smithi156 ceph-mon[98743]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:56.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:56 smithi156 ceph-mon[103290]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:56.416 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:56 smithi171 ceph-mon[90566]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:57.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:57 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3140810540' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:57.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:57 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3140810540' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:57.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:57 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3140810540' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:49:58.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:49:58 smithi156 ceph-mon[103290]: pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:58.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:49:58 smithi156 ceph-mon[98743]: pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:58.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:49:58 smithi171 ceph-mon[90566]: pgmap v167: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:00.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:00 smithi156 ceph-mon[103290]: pgmap v168: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:00.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:00 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3828970397' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:00.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:00 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:00.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:00 smithi156 ceph-mon[103290]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:50:00.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:00 smithi156 ceph-mon[98743]: pgmap v168: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:00.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:00 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3828970397' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:00.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:00 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:00.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:00 smithi156 ceph-mon[98743]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:50:00.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:00 smithi171 ceph-mon[90566]: pgmap v168: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:00.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:00 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3828970397' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:00.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:00 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:00.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:00 smithi171 ceph-mon[90566]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:50:02.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:02 smithi156 ceph-mon[98743]: pgmap v169: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:02.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:02 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2741199723' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:02.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:02 smithi156 ceph-mon[103290]: pgmap v169: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:02.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:02 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2741199723' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:02.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:02 smithi171 ceph-mon[90566]: pgmap v169: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:02.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:02 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2741199723' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:04.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:04 smithi156 ceph-mon[98743]: pgmap v170: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:04.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:04 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3376584368' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:04.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:04 smithi156 ceph-mon[103290]: pgmap v170: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:04.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:04 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3376584368' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:04.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:04 smithi171 ceph-mon[90566]: pgmap v170: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:04.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:04 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3376584368' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:05.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:50:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:50:04] "GET /metrics HTTP/1.1" 200 33698 "" "Prometheus/2.43.0" 2024-04-26T04:50:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:05 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:06.288 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:05 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:06.288 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:05 smithi171 ceph-mon[90566]: pgmap v171: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:06.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:06.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:06.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:05 smithi156 ceph-mon[98743]: pgmap v171: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:05 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:05 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:06.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:05 smithi156 ceph-mon[103290]: pgmap v171: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:07.078 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:06 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/4225166328' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:07.078 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:06 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:50:07.078 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:06 smithi156 ceph-mon[98743]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:50:07.078 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:06 smithi156 ceph-mon[98743]: Upgrade: osd.1 is safe to restart 2024-04-26T04:50:07.078 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:06 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/4225166328' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:07.078 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:06 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:50:07.078 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:06 smithi156 ceph-mon[103290]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:50:07.078 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:06 smithi156 ceph-mon[103290]: Upgrade: osd.1 is safe to restart 2024-04-26T04:50:07.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:06 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/4225166328' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:07.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:06 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:50:07.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:06 smithi171 ceph-mon[90566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:50:07.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:06 smithi171 ceph-mon[90566]: Upgrade: osd.1 is safe to restart 2024-04-26T04:50:09.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:08 smithi156 ceph-mon[98743]: pgmap v172: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:09.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:08 smithi156 ceph-mon[98743]: Upgrade: Updating osd.1 2024-04-26T04:50:09.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:08 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:09.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:08 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:50:09.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:08 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:09.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:08 smithi156 ceph-mon[98743]: Deploying daemon osd.1 on smithi156 2024-04-26T04:50:09.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:08 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1983470515' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:09.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:08 smithi156 ceph-mon[103290]: pgmap v172: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:09.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:08 smithi156 ceph-mon[103290]: Upgrade: Updating osd.1 2024-04-26T04:50:09.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:08 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:09.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:08 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:50:09.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:08 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:09.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:08 smithi156 ceph-mon[103290]: Deploying daemon osd.1 on smithi156 2024-04-26T04:50:09.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:08 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1983470515' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:09.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:08 smithi171 ceph-mon[90566]: pgmap v172: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:09.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:08 smithi171 ceph-mon[90566]: Upgrade: Updating osd.1 2024-04-26T04:50:09.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:08 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:09.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:08 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:50:09.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:08 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:09.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:08 smithi171 ceph-mon[90566]: Deploying daemon osd.1 on smithi156 2024-04-26T04:50:09.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:08 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1983470515' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:11.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:10 smithi156 ceph-mon[98743]: pgmap v173: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:11.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:10 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1503295737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:11.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:10 smithi156 ceph-mon[103290]: pgmap v173: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:11.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:10 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1503295737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:11.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:10 smithi171 ceph-mon[90566]: pgmap v173: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:11.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:10 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1503295737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:11.396 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:11 smithi156 systemd[1]: Stopping Ceph osd.1 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:50:11.843 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:11 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1[49950]: 2024-04-26T04:50:11.394+0000 7ff04d2f5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:50:11.843 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:11 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1[49950]: 2024-04-26T04:50:11.394+0000 7ff04d2f5700 -1 osd.1 83 *** Got signal Terminated *** 2024-04-26T04:50:11.843 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:11 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1[49950]: 2024-04-26T04:50:11.394+0000 7ff04d2f5700 -1 osd.1 83 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:50:12.145 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:11 smithi156 ceph-mon[98743]: osd.1 marked itself down and dead 2024-04-26T04:50:12.145 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:11 smithi156 ceph-mon[103290]: osd.1 marked itself down and dead 2024-04-26T04:50:12.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:11 smithi171 ceph-mon[90566]: osd.1 marked itself down and dead 2024-04-26T04:50:12.406 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:12 smithi156 podman[117478]: 2024-04-26 04:50:12.376141209 +0000 UTC m=+1.078105077 container died 7abca656e994508f14c741b444e3a92867dbe64276d1b1e768395ebb01d75ee9 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T04:50:13.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:12 smithi156 ceph-mon[98743]: pgmap v174: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:13.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:12 smithi156 ceph-mon[98743]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:50:13.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:12 smithi156 ceph-mon[98743]: osdmap e84: 8 total, 7 up, 8 in 2024-04-26T04:50:13.094 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:12 smithi156 podman[117478]: 2024-04-26 04:50:12.851360246 +0000 UTC m=+1.553324125 container cleanup 7abca656e994508f14c741b444e3a92867dbe64276d1b1e768395ebb01d75ee9 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:50:13.094 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:12 smithi156 bash[117478]: ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1 2024-04-26T04:50:13.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:12 smithi156 ceph-mon[103290]: pgmap v174: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:13.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:12 smithi156 ceph-mon[103290]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:50:13.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:12 smithi156 ceph-mon[103290]: osdmap e84: 8 total, 7 up, 8 in 2024-04-26T04:50:13.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:12 smithi171 ceph-mon[90566]: pgmap v174: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:13.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:12 smithi171 ceph-mon[90566]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:50:13.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:12 smithi171 ceph-mon[90566]: osdmap e84: 8 total, 7 up, 8 in 2024-04-26T04:50:13.421 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:13 smithi156 podman[117502]: 2024-04-26 04:50:13.111251011 +0000 UTC m=+0.730808098 container remove 7abca656e994508f14c741b444e3a92867dbe64276d1b1e768395ebb01d75ee9 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD) 2024-04-26T04:50:13.749 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:13 smithi156 podman[117631]: 2024-04-26 04:50:13.345386936 +0000 UTC m=+0.018128040 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:50:13.749 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:13 smithi156 podman[117631]: 2024-04-26 04:50:13.470797351 +0000 UTC m=+0.143538437 container create 1e78332570c66c81a12bf4586e9bc3dc2b42709f78d10dd14c34cf187cc285e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-26T04:50:14.055 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:13 smithi156 ceph-mon[103290]: osdmap e85: 8 total, 7 up, 8 in 2024-04-26T04:50:14.055 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:13 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1174101224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:14.055 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:13 smithi156 podman[117631]: 2024-04-26 04:50:13.748809091 +0000 UTC m=+0.421550184 container init 1e78332570c66c81a12bf4586e9bc3dc2b42709f78d10dd14c34cf187cc285e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True) 2024-04-26T04:50:14.055 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:13 smithi156 podman[117631]: 2024-04-26 04:50:13.754938428 +0000 UTC m=+0.427679515 container start 1e78332570c66c81a12bf4586e9bc3dc2b42709f78d10dd14c34cf187cc285e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-deactivate, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True) 2024-04-26T04:50:14.055 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:13 smithi156 podman[117631]: 2024-04-26 04:50:13.840189052 +0000 UTC m=+0.512930148 container attach 1e78332570c66c81a12bf4586e9bc3dc2b42709f78d10dd14c34cf187cc285e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-deactivate, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:50:14.055 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:13 smithi156 ceph-mon[98743]: osdmap e85: 8 total, 7 up, 8 in 2024-04-26T04:50:14.055 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:13 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1174101224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:14.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:13 smithi171 ceph-mon[90566]: osdmap e85: 8 total, 7 up, 8 in 2024-04-26T04:50:14.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:13 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1174101224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:14.343 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:14 smithi156 podman[117631]: 2024-04-26 04:50:14.051137078 +0000 UTC m=+0.723878170 container died 1e78332570c66c81a12bf4586e9bc3dc2b42709f78d10dd14c34cf187cc285e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-26T04:50:14.897 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:50:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:50:14] "GET /metrics HTTP/1.1" 200 33845 "" "Prometheus/2.43.0" 2024-04-26T04:50:14.897 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:14 smithi156 podman[117678]: 2024-04-26 04:50:14.635632763 +0000 UTC m=+0.578989309 container remove 1e78332570c66c81a12bf4586e9bc3dc2b42709f78d10dd14c34cf187cc285e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-deactivate, RELEASE=squid-b22e2eb, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-26T04:50:14.897 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:14 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-26T04:50:14.897 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:14 smithi156 systemd[1]: Stopped Ceph osd.1 for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:50:14.897 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:14 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.1.service: Consumed 5.817s CPU time. 2024-04-26T04:50:14.897 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:14 smithi156 systemd[1]: Starting Ceph osd.1 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:50:15.232 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:14 smithi156 ceph-mon[98743]: pgmap v177: 73 pgs: 8 stale+active+clean, 65 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:15.232 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:14 smithi156 ceph-mon[103290]: pgmap v177: 73 pgs: 8 stale+active+clean, 65 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:15.232 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 podman[117787]: 2024-04-26 04:50:14.989083995 +0000 UTC m=+0.018528975 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:50:15.232 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 podman[117787]: 2024-04-26 04:50:15.10330538 +0000 UTC m=+0.132750347 container create 00ce6b7261a37dcaad262766a877c9ad386f1f80576a3fff059434c5462da2df (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:50:15.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:14 smithi171 ceph-mon[90566]: pgmap v177: 73 pgs: 8 stale+active+clean, 65 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:15.593 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 podman[117787]: 2024-04-26 04:50:15.347502586 +0000 UTC m=+0.376947549 container init 00ce6b7261a37dcaad262766a877c9ad386f1f80576a3fff059434c5462da2df (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.build-date=20240424, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-04-26T04:50:15.593 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 podman[117787]: 2024-04-26 04:50:15.35360563 +0000 UTC m=+0.383050597 container start 00ce6b7261a37dcaad262766a877c9ad386f1f80576a3fff059434c5462da2df (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD) 2024-04-26T04:50:15.593 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 podman[117787]: 2024-04-26 04:50:15.438994362 +0000 UTC m=+0.468439332 container attach 00ce6b7261a37dcaad262766a877c9ad386f1f80576a3fff059434c5462da2df (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-activate, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-26T04:50:16.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:15 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/731418556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:16.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:15 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/731418556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:16.094 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-activate[117808]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T04:50:16.094 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-activate[117808]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T04:50:16.094 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 bash[117787]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T04:50:16.094 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 bash[117787]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T04:50:16.095 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 bash[117787]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T04:50:16.095 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 bash[117787]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T04:50:16.095 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 bash[117787]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-26T04:50:16.095 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 bash[117787]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T04:50:16.095 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 bash[117787]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-26T04:50:16.095 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-activate[117808]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T04:50:16.095 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-activate[117808]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T04:50:16.095 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-activate[117808]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-26T04:50:16.095 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-activate[117808]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T04:50:16.097 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-activate[117808]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-26T04:50:16.097 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:15 smithi156 podman[117787]: 2024-04-26 04:50:15.822326026 +0000 UTC m=+0.851770992 container died 00ce6b7261a37dcaad262766a877c9ad386f1f80576a3fff059434c5462da2df (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-26T04:50:16.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:15 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/731418556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:16.760 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:16 smithi156 podman[117976]: 2024-04-26 04:50:16.426454049 +0000 UTC m=+0.599109229 container remove 00ce6b7261a37dcaad262766a877c9ad386f1f80576a3fff059434c5462da2df (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_CLEAN=True) 2024-04-26T04:50:16.761 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:16 smithi156 podman[118043]: 2024-04-26 04:50:16.54165219 +0000 UTC m=+0.019060698 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:50:16.761 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:16 smithi156 podman[118043]: 2024-04-26 04:50:16.652516596 +0000 UTC m=+0.129925149 container create 3e65e778dd5a67580d11dd2acdcbd163c6c59d275864a558de9af8f7fc9c7736 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:50:17.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:16 smithi171 ceph-mon[90566]: pgmap v178: 73 pgs: 8 stale+active+clean, 65 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:17.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:16 smithi156 ceph-mon[98743]: pgmap v178: 73 pgs: 8 stale+active+clean, 65 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:17.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:16 smithi156 ceph-mon[103290]: pgmap v178: 73 pgs: 8 stale+active+clean, 65 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:17.343 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:16 smithi156 podman[118043]: 2024-04-26 04:50:16.888748982 +0000 UTC m=+0.366157493 container init 3e65e778dd5a67580d11dd2acdcbd163c6c59d275864a558de9af8f7fc9c7736 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-26T04:50:17.344 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:16 smithi156 podman[118043]: 2024-04-26 04:50:16.894322592 +0000 UTC m=+0.371731101 container start 3e65e778dd5a67580d11dd2acdcbd163c6c59d275864a558de9af8f7fc9c7736 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2) 2024-04-26T04:50:17.344 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:16 smithi156 ceph-osd[118080]: -- 172.21.15.156:0/871744717 <== mon.0 v2:172.21.15.156:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55999ed19680 con 0x55999ece3400 2024-04-26T04:50:17.344 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:16 smithi156 bash[118043]: 3e65e778dd5a67580d11dd2acdcbd163c6c59d275864a558de9af8f7fc9c7736 2024-04-26T04:50:17.344 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:16 smithi156 systemd[1]: Started Ceph osd.1 for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:50:17.843 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1[118057]: 2024-04-26T04:50:17.434+0000 7f506ff047c0 -1 Falling back to public interface 2024-04-26T04:50:18.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:18 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:18.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:18 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:18.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:18 smithi156 ceph-mon[103290]: pgmap v179: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:18.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:18 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:18.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:18 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/633392483' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:18.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:18 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:18.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:18 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:18.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:18 smithi156 ceph-mon[98743]: pgmap v179: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:18.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:18 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:18.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:18 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/633392483' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:18.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:18 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:18.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:18 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:18.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:18 smithi171 ceph-mon[90566]: pgmap v179: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:18.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:18 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:18.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:18 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/633392483' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:19.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:19 smithi156 ceph-mon[98743]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T04:50:19.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:19 smithi156 ceph-mon[103290]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T04:50:19.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:19 smithi171 ceph-mon[90566]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T04:50:20.343 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:20 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1[118057]: 2024-04-26T04:50:20.152+0000 7f506ff047c0 -1 osd.1 83 log_to_monitors true 2024-04-26T04:50:20.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:20 smithi156 ceph-mon[98743]: pgmap v180: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:20.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:20.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:20.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:20 smithi156 ceph-mon[103290]: pgmap v180: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:20.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:20.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:20.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:20 smithi171 ceph-mon[90566]: pgmap v180: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:20.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:20.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:21.333 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:21 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/58426291' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:21.333 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:21 smithi156 ceph-mon[98743]: from='osd.1 [v2:172.21.15.156:6810/915951660,v1:172.21.15.156:6811/915951660]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:50:21.333 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:21 smithi156 ceph-mon[98743]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:50:21.333 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:21 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:21.333 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:21 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:21.334 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:21 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/58426291' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:21.334 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:21 smithi156 ceph-mon[103290]: from='osd.1 [v2:172.21.15.156:6810/915951660,v1:172.21.15.156:6811/915951660]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:50:21.334 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:21 smithi156 ceph-mon[103290]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:50:21.334 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:21 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:21.334 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:21 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:21.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:21 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/58426291' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:21.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:21 smithi171 ceph-mon[90566]: from='osd.1 [v2:172.21.15.156:6810/915951660,v1:172.21.15.156:6811/915951660]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:50:21.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:21 smithi171 ceph-mon[90566]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:50:21.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:21 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:21.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:21 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:21.593 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:50:21 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1[118057]: 2024-04-26T04:50:21.332+0000 7f505d354700 -1 osd.1 83 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:50:22.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:22 smithi156 ceph-mon[98743]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:50:22.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:22 smithi156 ceph-mon[98743]: pgmap v181: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:22.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:22 smithi156 ceph-mon[98743]: osdmap e86: 8 total, 7 up, 8 in 2024-04-26T04:50:22.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:22 smithi156 ceph-mon[98743]: from='osd.1 [v2:172.21.15.156:6810/915951660,v1:172.21.15.156:6811/915951660]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:50:22.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:22 smithi156 ceph-mon[98743]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:50:22.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:22 smithi156 ceph-mon[98743]: from='osd.1 ' entity='osd.1' 2024-04-26T04:50:22.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:22 smithi156 ceph-mon[103290]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:50:22.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:22 smithi156 ceph-mon[103290]: pgmap v181: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:22.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:22 smithi156 ceph-mon[103290]: osdmap e86: 8 total, 7 up, 8 in 2024-04-26T04:50:22.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:22 smithi156 ceph-mon[103290]: from='osd.1 [v2:172.21.15.156:6810/915951660,v1:172.21.15.156:6811/915951660]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:50:22.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:22 smithi156 ceph-mon[103290]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:50:22.347 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:22 smithi156 ceph-mon[103290]: from='osd.1 ' entity='osd.1' 2024-04-26T04:50:22.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:22 smithi171 ceph-mon[90566]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:50:22.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:22 smithi171 ceph-mon[90566]: pgmap v181: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:22.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:22 smithi171 ceph-mon[90566]: osdmap e86: 8 total, 7 up, 8 in 2024-04-26T04:50:22.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:22 smithi171 ceph-mon[90566]: from='osd.1 [v2:172.21.15.156:6810/915951660,v1:172.21.15.156:6811/915951660]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:50:22.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:22 smithi171 ceph-mon[90566]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:50:22.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:22 smithi171 ceph-mon[90566]: from='osd.1 ' entity='osd.1' 2024-04-26T04:50:23.488 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:23 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:23.488 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:23 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:23.488 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:23 smithi156 ceph-mon[103290]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:50:23.488 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:23 smithi156 ceph-mon[103290]: osd.1 [v2:172.21.15.156:6810/915951660,v1:172.21.15.156:6811/915951660] boot 2024-04-26T04:50:23.489 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:23 smithi156 ceph-mon[103290]: osdmap e87: 8 total, 8 up, 8 in 2024-04-26T04:50:23.489 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:23 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:50:23.489 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:23 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3584932480' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:23.489 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:23 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:23.489 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:23 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:23.489 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:23 smithi156 ceph-mon[98743]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:50:23.489 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:23 smithi156 ceph-mon[98743]: osd.1 [v2:172.21.15.156:6810/915951660,v1:172.21.15.156:6811/915951660] boot 2024-04-26T04:50:23.489 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:23 smithi156 ceph-mon[98743]: osdmap e87: 8 total, 8 up, 8 in 2024-04-26T04:50:23.489 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:23 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:50:23.489 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:23 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3584932480' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:23.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:23 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:23.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:23 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:23.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:23 smithi171 ceph-mon[90566]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:50:23.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:23 smithi171 ceph-mon[90566]: osd.1 [v2:172.21.15.156:6810/915951660,v1:172.21.15.156:6811/915951660] boot 2024-04-26T04:50:23.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:23 smithi171 ceph-mon[90566]: osdmap e87: 8 total, 8 up, 8 in 2024-04-26T04:50:23.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:23 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:50:23.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:23 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3584932480' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:24.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:24 smithi156 ceph-mon[98743]: pgmap v184: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:24.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:24 smithi156 ceph-mon[98743]: osdmap e88: 8 total, 8 up, 8 in 2024-04-26T04:50:24.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:24 smithi156 ceph-mon[103290]: pgmap v184: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:24.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:24 smithi156 ceph-mon[103290]: osdmap e88: 8 total, 8 up, 8 in 2024-04-26T04:50:24.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:24 smithi171 ceph-mon[90566]: pgmap v184: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:24.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:24 smithi171 ceph-mon[90566]: osdmap e88: 8 total, 8 up, 8 in 2024-04-26T04:50:25.000 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:50:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:50:24] "GET /metrics HTTP/1.1" 200 33913 "" "Prometheus/2.43.0" 2024-04-26T04:50:25.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:25 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2131873240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:25.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:25 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2131873240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:25.651 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:25 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2131873240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:26.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:50:25 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:50:25.866+0000 7f741c426700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-26T04:50:26.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:26 smithi171 ceph-mon[90566]: pgmap v186: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:27.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:26 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:27.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:26 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:27.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:26 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:27.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:26 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:27.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:26 smithi171 ceph-mon[90566]: pgmap v187: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:27.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:26 smithi171 ceph-mon[90566]: pgmap v188: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:27.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:26 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:27.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:26 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:27.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:26 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:50:27.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:26 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:50:27.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:26 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:50:27.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:26 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:50:27.001 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:26 smithi171 ceph-mon[90566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:50:27.001 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:26 smithi171 ceph-mon[90566]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-26T04:50:27.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[98743]: pgmap v186: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:27.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:27.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:27.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:27.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:27.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[98743]: pgmap v187: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:27.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[98743]: pgmap v188: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:27.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:27.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:27.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:50:27.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:50:27.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:50:27.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:50:27.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[98743]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:50:27.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[98743]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-26T04:50:27.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[103290]: pgmap v186: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:27.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:27.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:27.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:27.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:27.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[103290]: pgmap v187: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:27.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[103290]: pgmap v188: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:50:27.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:27.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:27.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:50:27.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:50:27.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:50:27.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:50:27.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[103290]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:50:27.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:26 smithi156 ceph-mon[103290]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-26T04:50:27.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:27 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1340520047' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:28.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:27 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1340520047' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:28.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:27 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1340520047' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:28.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:28 smithi171 ceph-mon[90566]: pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:29.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:28 smithi156 ceph-mon[98743]: pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:29.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:28 smithi156 ceph-mon[103290]: pgmap v189: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:29.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:29 smithi171 ceph-mon[90566]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded) 2024-04-26T04:50:29.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:29 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1150787715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:30.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:29 smithi156 ceph-mon[103290]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded) 2024-04-26T04:50:30.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:29 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1150787715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:30.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:29 smithi156 ceph-mon[98743]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded) 2024-04-26T04:50:30.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:29 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1150787715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:30.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:30 smithi171 ceph-mon[90566]: pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:31.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:30 smithi156 ceph-mon[98743]: pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:31.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:30 smithi156 ceph-mon[103290]: pgmap v190: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:32.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:32 smithi171 ceph-mon[90566]: pgmap v191: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:33.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:32 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2610576316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:33.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:32 smithi156 ceph-mon[98743]: pgmap v191: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:33.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:32 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2610576316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:33.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:32 smithi156 ceph-mon[103290]: pgmap v191: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:33.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:32 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2610576316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:34.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:34 smithi171 ceph-mon[90566]: pgmap v192: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:35.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:34 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1028320885' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:35.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:50:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:50:34] "GET /metrics HTTP/1.1" 200 33913 "" "Prometheus/2.43.0" 2024-04-26T04:50:35.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:34 smithi156 ceph-mon[103290]: pgmap v192: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:35.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:34 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1028320885' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:35.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:34 smithi156 ceph-mon[98743]: pgmap v192: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:35.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:34 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1028320885' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:36.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:35 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:36.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:35 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:36.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:35 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:36.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:36.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:36.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:36.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:36.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:36.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:37.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:36 smithi171 ceph-mon[90566]: pgmap v193: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:37.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:36 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1061701323' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:37.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:36 smithi156 ceph-mon[98743]: pgmap v193: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:37.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:36 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1061701323' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:37.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:36 smithi156 ceph-mon[103290]: pgmap v193: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:37.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:36 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1061701323' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:38.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:37 smithi171 ceph-mon[90566]: pgmap v194: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:38.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:37 smithi156 ceph-mon[98743]: pgmap v194: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:38.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:37 smithi156 ceph-mon[103290]: pgmap v194: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:39.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:38 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3078137836' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:39.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:38 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3078137836' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:39.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:38 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3078137836' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:40.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:39 smithi171 ceph-mon[90566]: pgmap v195: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:40.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:39 smithi156 ceph-mon[98743]: pgmap v195: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:40.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:39 smithi156 ceph-mon[103290]: pgmap v195: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:41.244 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:40 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:50:41.244 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:40 smithi156 ceph-mon[98743]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:50:41.244 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:40 smithi156 ceph-mon[98743]: Upgrade: osd.2 is safe to restart 2024-04-26T04:50:41.244 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:40 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:50:41.244 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:40 smithi156 ceph-mon[103290]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:50:41.244 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:40 smithi156 ceph-mon[103290]: Upgrade: osd.2 is safe to restart 2024-04-26T04:50:41.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:40 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:50:41.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:40 smithi171 ceph-mon[90566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:50:41.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:40 smithi171 ceph-mon[90566]: Upgrade: osd.2 is safe to restart 2024-04-26T04:50:42.119 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:41 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/180320938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:42.138 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:41 smithi156 ceph-mon[98743]: pgmap v196: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:42.138 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:41 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/180320938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:42.138 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:41 smithi156 ceph-mon[103290]: pgmap v196: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:42.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:41 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/180320938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:42.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:41 smithi171 ceph-mon[90566]: pgmap v196: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:43.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:43 smithi171 ceph-mon[90566]: Upgrade: Updating osd.2 2024-04-26T04:50:43.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:43 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:43.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:43 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:50:43.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:43 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:43.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:43 smithi171 ceph-mon[90566]: Deploying daemon osd.2 on smithi156 2024-04-26T04:50:43.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:43 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/51259126' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:43.835 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:43 smithi156 ceph-mon[98743]: Upgrade: Updating osd.2 2024-04-26T04:50:43.835 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:43 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:43.835 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:43 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:50:43.835 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:43 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:43.835 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:43 smithi156 ceph-mon[98743]: Deploying daemon osd.2 on smithi156 2024-04-26T04:50:43.835 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:43 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/51259126' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:43.835 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:43 smithi156 ceph-mon[103290]: Upgrade: Updating osd.2 2024-04-26T04:50:43.835 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:43 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:43.836 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:43 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:50:43.836 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:43 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:43.836 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:43 smithi156 ceph-mon[103290]: Deploying daemon osd.2 on smithi156 2024-04-26T04:50:43.836 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:43 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/51259126' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:44.704 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:44 smithi156 ceph-mon[98743]: pgmap v197: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:44.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:44 smithi156 ceph-mon[103290]: pgmap v197: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:44.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:44 smithi171 ceph-mon[90566]: pgmap v197: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:45.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:50:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:50:44] "GET /metrics HTTP/1.1" 200 33906 "" "Prometheus/2.43.0" 2024-04-26T04:50:45.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:45 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1264931549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:46.068 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:45 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1264931549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:46.068 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:45 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1264931549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:46.068 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:45 smithi156 systemd[1]: Stopping Ceph osd.2 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:50:46.343 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:46 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2[56200]: 2024-04-26T04:50:46.066+0000 7f008f8ac700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:50:46.343 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:46 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2[56200]: 2024-04-26T04:50:46.066+0000 7f008f8ac700 -1 osd.2 88 *** Got signal Terminated *** 2024-04-26T04:50:46.343 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:46 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2[56200]: 2024-04-26T04:50:46.066+0000 7f008f8ac700 -1 osd.2 88 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:50:47.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:46 smithi156 ceph-mon[103290]: pgmap v198: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:47.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:46 smithi156 ceph-mon[103290]: osd.2 marked itself down and dead 2024-04-26T04:50:47.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:46 smithi156 ceph-mon[98743]: pgmap v198: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:47.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:46 smithi156 ceph-mon[98743]: osd.2 marked itself down and dead 2024-04-26T04:50:47.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:46 smithi171 ceph-mon[90566]: pgmap v198: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:47.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:46 smithi171 ceph-mon[90566]: osd.2 marked itself down and dead 2024-04-26T04:50:47.440 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:47 smithi156 podman[123046]: 2024-04-26 04:50:47.189839037 +0000 UTC m=+1.223745275 container died c951e2171c5dfc3ae8d239de094ffffe27125931fc99c996d3af9571897f4723 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0) 2024-04-26T04:50:47.707 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:47 smithi156 podman[123046]: 2024-04-26 04:50:47.522057153 +0000 UTC m=+1.555963398 container cleanup c951e2171c5dfc3ae8d239de094ffffe27125931fc99c996d3af9571897f4723 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_CLEAN=True) 2024-04-26T04:50:47.707 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:47 smithi156 bash[123046]: ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2 2024-04-26T04:50:48.093 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:47 smithi156 podman[123068]: 2024-04-26 04:50:47.706539095 +0000 UTC m=+0.519518177 container remove c951e2171c5dfc3ae8d239de094ffffe27125931fc99c996d3af9571897f4723 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-26T04:50:48.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:47 smithi156 ceph-mon[98743]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:50:48.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:47 smithi156 ceph-mon[98743]: osdmap e89: 8 total, 7 up, 8 in 2024-04-26T04:50:48.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:47 smithi156 ceph-mon[98743]: osdmap e90: 8 total, 7 up, 8 in 2024-04-26T04:50:48.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:47 smithi156 ceph-mon[103290]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:50:48.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:47 smithi156 ceph-mon[103290]: osdmap e89: 8 total, 7 up, 8 in 2024-04-26T04:50:48.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:47 smithi156 ceph-mon[103290]: osdmap e90: 8 total, 7 up, 8 in 2024-04-26T04:50:48.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:47 smithi171 ceph-mon[90566]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:50:48.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:47 smithi171 ceph-mon[90566]: osdmap e89: 8 total, 7 up, 8 in 2024-04-26T04:50:48.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:47 smithi171 ceph-mon[90566]: osdmap e90: 8 total, 7 up, 8 in 2024-04-26T04:50:48.433 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:48 smithi156 podman[123184]: 2024-04-26 04:50:48.084561006 +0000 UTC m=+0.019003159 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:50:48.433 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:48 smithi156 podman[123184]: 2024-04-26 04:50:48.207820326 +0000 UTC m=+0.142262461 container create f5b440321b0d7d32f7b2c6b521d01da2727ee88c8ae86ca54fcd898fb0a8e5f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-26T04:50:48.433 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:48 smithi156 podman[123184]: 2024-04-26 04:50:48.42678594 +0000 UTC m=+0.361228075 container init f5b440321b0d7d32f7b2c6b521d01da2727ee88c8ae86ca54fcd898fb0a8e5f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:50:48.724 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:48 smithi156 podman[123184]: 2024-04-26 04:50:48.432570495 +0000 UTC m=+0.367012631 container start f5b440321b0d7d32f7b2c6b521d01da2727ee88c8ae86ca54fcd898fb0a8e5f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-26T04:50:48.724 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:48 smithi156 podman[123184]: 2024-04-26 04:50:48.518403521 +0000 UTC m=+0.452845670 container attach f5b440321b0d7d32f7b2c6b521d01da2727ee88c8ae86ca54fcd898fb0a8e5f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True) 2024-04-26T04:50:49.093 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:48 smithi156 podman[123184]: 2024-04-26 04:50:48.724277767 +0000 UTC m=+0.658719897 container died f5b440321b0d7d32f7b2c6b521d01da2727ee88c8ae86ca54fcd898fb0a8e5f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True) 2024-04-26T04:50:49.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:48 smithi156 ceph-mon[98743]: pgmap v200: 73 pgs: 4 stale+active+clean, 69 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:49.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:48 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3436736678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:49.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:48 smithi156 ceph-mon[103290]: pgmap v200: 73 pgs: 4 stale+active+clean, 69 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:49.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:48 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3436736678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:49.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:48 smithi171 ceph-mon[90566]: pgmap v200: 73 pgs: 4 stale+active+clean, 69 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:49.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:48 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3436736678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:49.348 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:49 smithi156 podman[123232]: 2024-04-26 04:50:49.330773071 +0000 UTC m=+0.601208031 container remove f5b440321b0d7d32f7b2c6b521d01da2727ee88c8ae86ca54fcd898fb0a8e5f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-deactivate, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:50:49.773 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:49 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-26T04:50:49.773 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:49 smithi156 systemd[1]: Stopped Ceph osd.2 for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:50:49.773 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:49 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.2.service: Consumed 8.504s CPU time. 2024-04-26T04:50:49.773 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:49 smithi156 systemd[1]: Starting Ceph osd.2 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:50:50.060 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:49 smithi156 podman[123342]: 2024-04-26 04:50:49.676047867 +0000 UTC m=+0.018161161 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:50:50.060 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:49 smithi156 podman[123342]: 2024-04-26 04:50:49.815089908 +0000 UTC m=+0.157203177 container create 7177ef18f539b02a43a35f621d2bde8bc4061e6ad9f7292b586eb263e235f805 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-26T04:50:50.343 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 podman[123342]: 2024-04-26 04:50:50.059860784 +0000 UTC m=+0.401974059 container init 7177ef18f539b02a43a35f621d2bde8bc4061e6ad9f7292b586eb263e235f805 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-26T04:50:50.344 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 podman[123342]: 2024-04-26 04:50:50.065295569 +0000 UTC m=+0.407408838 container start 7177ef18f539b02a43a35f621d2bde8bc4061e6ad9f7292b586eb263e235f805 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-activate, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424) 2024-04-26T04:50:50.344 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 podman[123342]: 2024-04-26 04:50:50.134579256 +0000 UTC m=+0.476692529 container attach 7177ef18f539b02a43a35f621d2bde8bc4061e6ad9f7292b586eb263e235f805 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-26T04:50:50.840 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-activate[123356]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T04:50:50.840 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-activate[123356]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T04:50:50.840 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 bash[123342]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T04:50:50.840 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 bash[123342]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T04:50:50.840 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 bash[123342]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T04:50:50.840 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 bash[123342]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T04:50:50.840 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 bash[123342]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-26T04:50:50.840 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 bash[123342]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T04:50:50.840 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 bash[123342]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-26T04:50:50.840 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-activate[123356]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T04:50:50.840 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-activate[123356]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T04:50:50.841 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-activate[123356]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-26T04:50:50.841 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-activate[123356]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T04:50:50.841 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-activate[123356]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-26T04:50:50.841 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:50 smithi156 podman[123342]: 2024-04-26 04:50:50.541695103 +0000 UTC m=+0.883808380 container died 7177ef18f539b02a43a35f621d2bde8bc4061e6ad9f7292b586eb263e235f805 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:50:51.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:50 smithi156 ceph-mon[98743]: pgmap v202: 73 pgs: 4 stale+active+clean, 69 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:51.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:50 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:51.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:50 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:51.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:50 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3536044708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:51.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:50 smithi156 ceph-mon[103290]: pgmap v202: 73 pgs: 4 stale+active+clean, 69 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:51.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:50 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:51.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:50 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:51.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:50 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3536044708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:51.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:50 smithi171 ceph-mon[90566]: pgmap v202: 73 pgs: 4 stale+active+clean, 69 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:51.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:50 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:51.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:50 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:51.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:50 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3536044708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:51.558 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:51 smithi156 podman[123530]: 2024-04-26 04:50:51.130543444 +0000 UTC m=+0.579420964 container remove 7177ef18f539b02a43a35f621d2bde8bc4061e6ad9f7292b586eb263e235f805 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-activate, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-26T04:50:51.559 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:51 smithi156 podman[123599]: 2024-04-26 04:50:51.244878556 +0000 UTC m=+0.018365987 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:50:51.559 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:51 smithi156 podman[123599]: 2024-04-26 04:50:51.356671627 +0000 UTC m=+0.130159049 container create fd77f3d690838577cd22aa333ae2fadd385606cf4482b2f47a7c1bc536299ca3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-26T04:50:52.093 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:51 smithi156 podman[123599]: 2024-04-26 04:50:51.708907576 +0000 UTC m=+0.482394998 container init fd77f3d690838577cd22aa333ae2fadd385606cf4482b2f47a7c1bc536299ca3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True) 2024-04-26T04:50:52.093 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:51 smithi156 podman[123599]: 2024-04-26 04:50:51.714390178 +0000 UTC m=+0.487877603 container start fd77f3d690838577cd22aa333ae2fadd385606cf4482b2f47a7c1bc536299ca3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-26T04:50:52.093 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:51 smithi156 ceph-osd[123638]: -- 172.21.15.156:0/3573754575 <== mon.0 v2:172.21.15.156:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ede31f7680 con 0x55ede31c0c00 2024-04-26T04:50:52.093 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:51 smithi156 bash[123599]: fd77f3d690838577cd22aa333ae2fadd385606cf4482b2f47a7c1bc536299ca3 2024-04-26T04:50:52.093 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:51 smithi156 systemd[1]: Started Ceph osd.2 for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:50:52.593 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:52 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2[123614]: 2024-04-26T04:50:52.267+0000 7f08037b67c0 -1 Falling back to public interface 2024-04-26T04:50:52.988 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:52 smithi156 ceph-mon[98743]: pgmap v203: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:50:52.988 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:52 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:52.988 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:52 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:52.988 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:52 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:52.988 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:52 smithi156 ceph-mon[98743]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:50:52.988 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:52 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1381879549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:52.989 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:52 smithi156 ceph-mon[103290]: pgmap v203: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:50:52.989 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:52 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:52.989 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:52 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:52.989 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:52 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:52.989 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:52 smithi156 ceph-mon[103290]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:50:52.989 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:52 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1381879549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:52.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:52 smithi171 ceph-mon[90566]: pgmap v203: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:50:52.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:52 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:52.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:52 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:52.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:52 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:52.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:52 smithi171 ceph-mon[90566]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:50:52.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:52 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1381879549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:54.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:50:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:50:54] "GET /metrics HTTP/1.1" 200 33907 "" "Prometheus/2.43.0" 2024-04-26T04:50:55.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:54 smithi171 ceph-mon[90566]: pgmap v204: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:50:55.315 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:54 smithi156 ceph-mon[103290]: pgmap v204: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:50:55.315 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:54 smithi156 ceph-mon[98743]: pgmap v204: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:50:55.316 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2[123614]: 2024-04-26T04:50:54.974+0000 7f08037b67c0 -1 osd.2 88 log_to_monitors true 2024-04-26T04:50:55.878 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:55 smithi171 ceph-mon[90566]: from='osd.2 [v2:172.21.15.156:6818/3864636890,v1:172.21.15.156:6819/3864636890]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:50:55.878 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:55 smithi171 ceph-mon[90566]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:50:55.878 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:55 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3864214770' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:55.878 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:55 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:55.878 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:55 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:56.343 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:50:55 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2[123614]: 2024-04-26T04:50:55.895+0000 7f07f4c0e700 -1 osd.2 88 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:50:56.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:55 smithi156 ceph-mon[98743]: from='osd.2 [v2:172.21.15.156:6818/3864636890,v1:172.21.15.156:6819/3864636890]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:50:56.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:55 smithi156 ceph-mon[98743]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:50:56.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:55 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3864214770' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:56.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:55 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:56.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:55 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:56.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:55 smithi156 ceph-mon[103290]: from='osd.2 [v2:172.21.15.156:6818/3864636890,v1:172.21.15.156:6819/3864636890]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:50:56.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:55 smithi156 ceph-mon[103290]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:50:56.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:55 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3864214770' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:56.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:55 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:56.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:55 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:57.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:56 smithi171 ceph-mon[90566]: pgmap v205: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:50:57.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:56 smithi171 ceph-mon[90566]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:50:57.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:56 smithi171 ceph-mon[90566]: osdmap e91: 8 total, 7 up, 8 in 2024-04-26T04:50:57.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:56 smithi171 ceph-mon[90566]: from='osd.2 [v2:172.21.15.156:6818/3864636890,v1:172.21.15.156:6819/3864636890]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:50:57.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:56 smithi171 ceph-mon[90566]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:50:57.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:56 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:57.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:56 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:57.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:56 smithi156 ceph-mon[98743]: pgmap v205: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:50:57.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:56 smithi156 ceph-mon[98743]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:50:57.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:56 smithi156 ceph-mon[98743]: osdmap e91: 8 total, 7 up, 8 in 2024-04-26T04:50:57.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:56 smithi156 ceph-mon[98743]: from='osd.2 [v2:172.21.15.156:6818/3864636890,v1:172.21.15.156:6819/3864636890]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:50:57.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:56 smithi156 ceph-mon[98743]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:50:57.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:56 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:57.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:56 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:57.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:56 smithi156 ceph-mon[103290]: pgmap v205: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:50:57.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:56 smithi156 ceph-mon[103290]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:50:57.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:56 smithi156 ceph-mon[103290]: osdmap e91: 8 total, 7 up, 8 in 2024-04-26T04:50:57.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:56 smithi156 ceph-mon[103290]: from='osd.2 [v2:172.21.15.156:6818/3864636890,v1:172.21.15.156:6819/3864636890]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:50:57.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:56 smithi156 ceph-mon[103290]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:50:57.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:56 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:57.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:56 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:58.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:57 smithi171 ceph-mon[90566]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:50:58.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:57 smithi171 ceph-mon[90566]: osd.2 [v2:172.21.15.156:6818/3864636890,v1:172.21.15.156:6819/3864636890] boot 2024-04-26T04:50:58.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:57 smithi171 ceph-mon[90566]: osdmap e92: 8 total, 8 up, 8 in 2024-04-26T04:50:58.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:57 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:50:58.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:57 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:58.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:57 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:58.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:57 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3325169490' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:58.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:57 smithi171 ceph-mon[90566]: osdmap e93: 8 total, 8 up, 8 in 2024-04-26T04:50:58.310 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:57 smithi156 ceph-mon[98743]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:50:58.310 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:57 smithi156 ceph-mon[98743]: osd.2 [v2:172.21.15.156:6818/3864636890,v1:172.21.15.156:6819/3864636890] boot 2024-04-26T04:50:58.310 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:57 smithi156 ceph-mon[98743]: osdmap e92: 8 total, 8 up, 8 in 2024-04-26T04:50:58.310 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:57 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:50:58.310 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:57 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:58.310 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:57 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:58.310 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:57 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3325169490' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:58.310 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:57 smithi156 ceph-mon[98743]: osdmap e93: 8 total, 8 up, 8 in 2024-04-26T04:50:58.311 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:57 smithi156 ceph-mon[103290]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:50:58.311 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:57 smithi156 ceph-mon[103290]: osd.2 [v2:172.21.15.156:6818/3864636890,v1:172.21.15.156:6819/3864636890] boot 2024-04-26T04:50:58.311 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:57 smithi156 ceph-mon[103290]: osdmap e92: 8 total, 8 up, 8 in 2024-04-26T04:50:58.311 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:57 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:50:58.311 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:57 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:58.311 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:57 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:50:58.311 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:57 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3325169490' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:50:58.311 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:57 smithi156 ceph-mon[103290]: osdmap e93: 8 total, 8 up, 8 in 2024-04-26T04:50:59.181 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:58 smithi156 ceph-mon[98743]: pgmap v208: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:50:59.181 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:58 smithi156 ceph-mon[103290]: pgmap v208: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:50:59.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:58 smithi171 ceph-mon[90566]: pgmap v208: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:00.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:50:59 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3448688841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:00.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:50:59 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3448688841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:00.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:50:59 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3448688841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:00.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:51:00.647+0000 7f741c426700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-26T04:51:01.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:00 smithi171 ceph-mon[90566]: pgmap v210: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:01.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:00 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:01.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:00 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:01.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:00 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:01.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:00 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:01.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:00 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:01.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:00 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:01.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:00 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:01.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:00 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:01.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:00 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:01.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:00 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:51:01.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[103290]: pgmap v210: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:01.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:01.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:01.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:01.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:01.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:01.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:01.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:01.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:01.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:01.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:51:01.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[98743]: pgmap v210: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:01.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:01.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:01.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:01.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:01.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:01.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:01.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:01.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:01.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:01.347 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:00 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:51:02.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:01 smithi171 ceph-mon[90566]: pgmap v211: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:02.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:01 smithi171 ceph-mon[90566]: pgmap v212: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:02.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:01 smithi171 ceph-mon[90566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:51:02.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:01 smithi171 ceph-mon[90566]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-26T04:51:02.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:01 smithi156 ceph-mon[98743]: pgmap v211: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:02.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:01 smithi156 ceph-mon[98743]: pgmap v212: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:02.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:01 smithi156 ceph-mon[98743]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:51:02.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:01 smithi156 ceph-mon[98743]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-26T04:51:02.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:01 smithi156 ceph-mon[103290]: pgmap v211: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:02.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:01 smithi156 ceph-mon[103290]: pgmap v212: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:02.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:01 smithi156 ceph-mon[103290]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:51:02.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:01 smithi156 ceph-mon[103290]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-26T04:51:03.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:02 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/993052523' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:03.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:02 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/993052523' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:03.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:02 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/993052523' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:04.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:03 smithi171 ceph-mon[90566]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:04.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:03 smithi171 ceph-mon[90566]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded) 2024-04-26T04:51:04.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:03 smithi156 ceph-mon[98743]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:04.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:03 smithi156 ceph-mon[98743]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded) 2024-04-26T04:51:04.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:03 smithi156 ceph-mon[103290]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:04.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:03 smithi156 ceph-mon[103290]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded) 2024-04-26T04:51:04.972 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:51:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:51:04] "GET /metrics HTTP/1.1" 200 33907 "" "Prometheus/2.43.0" 2024-04-26T04:51:05.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:04 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/4180438549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:05.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:04 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/4180438549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:05.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:04 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/4180438549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:06.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:05 smithi171 ceph-mon[90566]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:06.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:05 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:06.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:05 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:06.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:05 smithi156 ceph-mon[103290]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:06.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:05 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:06.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:05 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:06.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:05 smithi156 ceph-mon[98743]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:06.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:06.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:07.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:06 smithi171 ceph-mon[90566]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:07.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:06 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2076851003' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:07.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:06 smithi156 ceph-mon[103290]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:07.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:06 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2076851003' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:07.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:06 smithi156 ceph-mon[98743]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:07.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:06 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2076851003' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:09.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:09 smithi156 ceph-mon[103290]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:09.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:09 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3175797383' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:09.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:09 smithi156 ceph-mon[98743]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:09.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:09 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3175797383' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:09.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:09 smithi171 ceph-mon[90566]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:09.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:09 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3175797383' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:11.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:11 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:11.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:11 smithi156 ceph-mon[98743]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:11.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:11 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:11.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:11 smithi156 ceph-mon[103290]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:11.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:11 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:11.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:11 smithi171 ceph-mon[90566]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:12.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:12 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2639672939' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:12.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:12 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2639672939' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:12.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:12 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2639672939' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:13.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:13 smithi156 ceph-mon[98743]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:13.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:13 smithi156 ceph-mon[103290]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:13.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:13 smithi171 ceph-mon[90566]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:14.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:14 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3778907883' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:14.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:14 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3778907883' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:14.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:14 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3778907883' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:15.015 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:51:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:51:14] "GET /metrics HTTP/1.1" 200 33901 "" "Prometheus/2.43.0" 2024-04-26T04:51:15.266 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:15 smithi156 ceph-mon[103290]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:15.267 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:15 smithi156 ceph-mon[98743]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:15.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:15 smithi171 ceph-mon[90566]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:16.295 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:16 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:51:16.295 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:16 smithi156 ceph-mon[98743]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:51:16.295 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:16 smithi156 ceph-mon[98743]: Upgrade: osd.3 is safe to restart 2024-04-26T04:51:16.295 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:16 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2722295932' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:16.296 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:16 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:51:16.296 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:16 smithi156 ceph-mon[103290]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:51:16.296 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:16 smithi156 ceph-mon[103290]: Upgrade: osd.3 is safe to restart 2024-04-26T04:51:16.296 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:16 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2722295932' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:16.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:16 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:51:16.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:16 smithi171 ceph-mon[90566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:51:16.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:16 smithi171 ceph-mon[90566]: Upgrade: osd.3 is safe to restart 2024-04-26T04:51:16.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:16 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2722295932' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:17.334 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:17 smithi156 ceph-mon[98743]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:17.334 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:17 smithi156 ceph-mon[103290]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:17.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:17 smithi171 ceph-mon[90566]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:18.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:18 smithi156 ceph-mon[98743]: Upgrade: Updating osd.3 2024-04-26T04:51:18.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:18 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:18.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:18 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:51:18.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:18 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:18.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:18 smithi156 ceph-mon[98743]: Deploying daemon osd.3 on smithi156 2024-04-26T04:51:18.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:18 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/658748099' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:18.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:18 smithi156 ceph-mon[103290]: Upgrade: Updating osd.3 2024-04-26T04:51:18.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:18 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:18.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:18 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:51:18.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:18 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:18.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:18 smithi156 ceph-mon[103290]: Deploying daemon osd.3 on smithi156 2024-04-26T04:51:18.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:18 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/658748099' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:18.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:18 smithi171 ceph-mon[90566]: Upgrade: Updating osd.3 2024-04-26T04:51:18.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:18 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:18.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:18 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:51:18.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:18 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:18.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:18 smithi171 ceph-mon[90566]: Deploying daemon osd.3 on smithi156 2024-04-26T04:51:18.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:18 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/658748099' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:19.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:19 smithi156 ceph-mon[98743]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:19.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:19 smithi156 ceph-mon[103290]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:19.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:19 smithi171 ceph-mon[90566]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:20.722 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:20.722 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:20.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:20.992 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:20 smithi156 systemd[1]: Stopping Ceph osd.3 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:51:21.343 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:20 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3[62345]: 2024-04-26T04:51:20.989+0000 7f749fd4b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:51:21.343 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:20 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3[62345]: 2024-04-26T04:51:20.989+0000 7f749fd4b700 -1 osd.3 93 *** Got signal Terminated *** 2024-04-26T04:51:21.343 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:20 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3[62345]: 2024-04-26T04:51:20.989+0000 7f749fd4b700 -1 osd.3 93 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:51:21.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:21 smithi171 ceph-mon[90566]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:21.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:21 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3992503106' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:21.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:21 smithi171 ceph-mon[90566]: osd.3 marked itself down and dead 2024-04-26T04:51:21.788 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:21 smithi156 ceph-mon[98743]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:21.788 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:21 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3992503106' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:21.788 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:21 smithi156 ceph-mon[98743]: osd.3 marked itself down and dead 2024-04-26T04:51:21.788 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:21 smithi156 ceph-mon[103290]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:21.788 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:21 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3992503106' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:21.788 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:21 smithi156 ceph-mon[103290]: osd.3 marked itself down and dead 2024-04-26T04:51:22.089 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:21 smithi156 podman[128635]: 2024-04-26 04:51:21.787323916 +0000 UTC m=+0.922332078 container died a2c01ca7770f45b396769c42dd5b13fc37c344b85f886ea64dbcc5d055ea90c2 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240424) 2024-04-26T04:51:22.343 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:22 smithi156 podman[128635]: 2024-04-26 04:51:22.271333577 +0000 UTC m=+1.406341747 container cleanup a2c01ca7770f45b396769c42dd5b13fc37c344b85f886ea64dbcc5d055ea90c2 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3, RELEASE=quincy-c61472c, ceph=True, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:51:22.343 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:22 smithi156 bash[128635]: ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3 2024-04-26T04:51:22.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:22 smithi171 ceph-mon[90566]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:51:22.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:22 smithi171 ceph-mon[90566]: osdmap e94: 8 total, 7 up, 8 in 2024-04-26T04:51:22.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:22 smithi171 ceph-mon[90566]: osdmap e95: 8 total, 7 up, 8 in 2024-04-26T04:51:22.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:22 smithi156 ceph-mon[98743]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:51:22.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:22 smithi156 ceph-mon[98743]: osdmap e94: 8 total, 7 up, 8 in 2024-04-26T04:51:22.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:22 smithi156 ceph-mon[98743]: osdmap e95: 8 total, 7 up, 8 in 2024-04-26T04:51:22.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:22 smithi156 ceph-mon[103290]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:51:22.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:22 smithi156 ceph-mon[103290]: osdmap e94: 8 total, 7 up, 8 in 2024-04-26T04:51:22.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:22 smithi156 ceph-mon[103290]: osdmap e95: 8 total, 7 up, 8 in 2024-04-26T04:51:22.844 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:22 smithi156 podman[128657]: 2024-04-26 04:51:22.531375943 +0000 UTC m=+0.739151703 container remove a2c01ca7770f45b396769c42dd5b13fc37c344b85f886ea64dbcc5d055ea90c2 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-26T04:51:23.168 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:22 smithi156 podman[128759]: 2024-04-26 04:51:22.760435939 +0000 UTC m=+0.018640572 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:23.169 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:22 smithi156 podman[128759]: 2024-04-26 04:51:22.890715763 +0000 UTC m=+0.148920361 container create ccea58e9441c138d82de7619baa1b4c549cd5c97bd7365dad3d3f00e4ec317fb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux ) 2024-04-26T04:51:23.169 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:23 smithi156 podman[128759]: 2024-04-26 04:51:23.084952017 +0000 UTC m=+0.343156616 container init ccea58e9441c138d82de7619baa1b4c549cd5c97bd7365dad3d3f00e4ec317fb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:51:23.169 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:23 smithi156 podman[128759]: 2024-04-26 04:51:23.090679443 +0000 UTC m=+0.348884042 container start ccea58e9441c138d82de7619baa1b4c549cd5c97bd7365dad3d3f00e4ec317fb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240424) 2024-04-26T04:51:23.169 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:23 smithi156 podman[128759]: 2024-04-26 04:51:23.168149461 +0000 UTC m=+0.426354061 container attach ccea58e9441c138d82de7619baa1b4c549cd5c97bd7365dad3d3f00e4ec317fb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240424) 2024-04-26T04:51:23.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:23 smithi156 ceph-mon[98743]: pgmap v225: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:23.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:23 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1778520447' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:23.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:23 smithi156 ceph-mon[103290]: pgmap v225: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:23.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:23 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1778520447' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:23.594 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:23 smithi156 conmon[128793]: conmon ccea58e9441c138d82de : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-ccea58e9441c138d82de7619baa1b4c549cd5c97bd7365dad3d3f00e4ec317fb.scope/memory.events 2024-04-26T04:51:23.594 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:23 smithi156 podman[128759]: 2024-04-26 04:51:23.387495436 +0000 UTC m=+0.645700046 container died ccea58e9441c138d82de7619baa1b4c549cd5c97bd7365dad3d3f00e4ec317fb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-26T04:51:23.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:23 smithi171 ceph-mon[90566]: pgmap v225: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:23.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:23 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1778520447' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:24.343 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:23 smithi156 podman[128827]: 2024-04-26 04:51:23.963774731 +0000 UTC m=+0.567161373 container remove ccea58e9441c138d82de7619baa1b4c549cd5c97bd7365dad3d3f00e4ec317fb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, ceph=True, GIT_CLEAN=True) 2024-04-26T04:51:24.343 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:23 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-26T04:51:24.343 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:23 smithi156 systemd[1]: Stopped Ceph osd.3 for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:51:24.343 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:23 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.3.service: Consumed 5.377s CPU time. 2024-04-26T04:51:24.343 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:24 smithi156 systemd[1]: Starting Ceph osd.3 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:51:24.703 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:24 smithi156 podman[128940]: 2024-04-26 04:51:24.304672304 +0000 UTC m=+0.019004686 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:24.703 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:24 smithi156 podman[128940]: 2024-04-26 04:51:24.423493353 +0000 UTC m=+0.137825729 container create 40fcf8971bb137432a7dbc4c481cc3c0882fffe24af3173c425638985e2c6c7c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0) 2024-04-26T04:51:24.703 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:24 smithi156 podman[128940]: 2024-04-26 04:51:24.643136759 +0000 UTC m=+0.357469140 container init 40fcf8971bb137432a7dbc4c481cc3c0882fffe24af3173c425638985e2c6c7c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=) 2024-04-26T04:51:24.703 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:24 smithi156 podman[128940]: 2024-04-26 04:51:24.648548283 +0000 UTC m=+0.362880657 container start 40fcf8971bb137432a7dbc4c481cc3c0882fffe24af3173c425638985e2c6c7c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-activate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-26T04:51:25.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:51:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:51:24] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.43.0" 2024-04-26T04:51:25.093 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:24 smithi156 podman[128940]: 2024-04-26 04:51:24.725832344 +0000 UTC m=+0.440164727 container attach 40fcf8971bb137432a7dbc4c481cc3c0882fffe24af3173c425638985e2c6c7c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-activate, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:51:25.497 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:25 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-activate[128955]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T04:51:25.497 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:25 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-activate[128955]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T04:51:25.497 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:25 smithi156 bash[128940]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T04:51:25.497 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:25 smithi156 bash[128940]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T04:51:25.497 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:25 smithi156 bash[128940]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T04:51:25.497 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:25 smithi156 bash[128940]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T04:51:25.497 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:25 smithi156 bash[128940]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-26T04:51:25.497 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:25 smithi156 bash[128940]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T04:51:25.498 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:25 smithi156 bash[128940]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-26T04:51:25.498 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:25 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-activate[128955]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T04:51:25.498 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:25 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-activate[128955]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T04:51:25.498 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:25 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-activate[128955]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-26T04:51:25.498 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:25 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-activate[128955]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T04:51:25.498 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:25 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-activate[128955]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-26T04:51:25.498 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:25 smithi156 podman[128940]: 2024-04-26 04:51:25.124084173 +0000 UTC m=+0.838416547 container died 40fcf8971bb137432a7dbc4c481cc3c0882fffe24af3173c425638985e2c6c7c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-26T04:51:25.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:25 smithi156 ceph-mon[98743]: pgmap v226: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:25.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:25 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/551583037' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:25.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:25 smithi156 ceph-mon[103290]: pgmap v226: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:25.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:25 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/551583037' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:25.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:25 smithi171 ceph-mon[90566]: pgmap v226: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:25.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:25 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/551583037' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:26.254 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:25 smithi156 podman[129125]: 2024-04-26 04:51:25.905590597 +0000 UTC m=+0.776203646 container remove 40fcf8971bb137432a7dbc4c481cc3c0882fffe24af3173c425638985e2c6c7c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:51:26.254 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:26 smithi156 podman[129198]: 2024-04-26 04:51:26.022919963 +0000 UTC m=+0.018270899 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:26.255 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:26 smithi156 podman[129198]: 2024-04-26 04:51:26.131665995 +0000 UTC m=+0.127016913 container create 698081f2c4f0d8fb6965b895cdfff1fd436f8a3bc8c95a1d81995f23fe364b6e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:51:26.843 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:26 smithi156 podman[129198]: 2024-04-26 04:51:26.375562046 +0000 UTC m=+0.370912965 container init 698081f2c4f0d8fb6965b895cdfff1fd436f8a3bc8c95a1d81995f23fe364b6e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:51:26.843 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:26 smithi156 podman[129198]: 2024-04-26 04:51:26.38097952 +0000 UTC m=+0.376330438 container start 698081f2c4f0d8fb6965b895cdfff1fd436f8a3bc8c95a1d81995f23fe364b6e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:51:26.843 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:26 smithi156 ceph-osd[129235]: -- 172.21.15.156:0/2086835573 <== mon.2 v2:172.21.15.156:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555d7618b680 con 0x555d76155000 2024-04-26T04:51:26.843 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:26 smithi156 bash[129198]: 698081f2c4f0d8fb6965b895cdfff1fd436f8a3bc8c95a1d81995f23fe364b6e 2024-04-26T04:51:26.843 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:26 smithi156 systemd[1]: Started Ceph osd.3 for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:51:27.343 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:26 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3[129212]: 2024-04-26T04:51:26.929+0000 7fa3114007c0 -1 Falling back to public interface 2024-04-26T04:51:27.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:27 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:27.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:27 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:27.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:27 smithi156 ceph-mon[98743]: pgmap v227: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:27.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:27 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:27.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:27 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:27.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:27 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:27.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:27 smithi156 ceph-mon[103290]: pgmap v227: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:27.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:27 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:27.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:27 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:27.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:27 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:27.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:27 smithi171 ceph-mon[90566]: pgmap v227: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:27.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:27 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:28.817 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:28 smithi156 ceph-mon[98743]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:28.817 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:28 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1361160987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:28.817 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:28 smithi156 ceph-mon[103290]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:28.817 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:28 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1361160987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:28.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:28 smithi171 ceph-mon[90566]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:28.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:28 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1361160987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:29.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:29 smithi156 ceph-mon[98743]: pgmap v228: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:29.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:29 smithi156 ceph-mon[103290]: pgmap v228: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:29.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:29 smithi171 ceph-mon[90566]: pgmap v228: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:30.057 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:29 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3[129212]: 2024-04-26T04:51:29.640+0000 7fa3114007c0 -1 osd.3 93 log_to_monitors true 2024-04-26T04:51:30.814 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:30 smithi156 ceph-mon[98743]: from='osd.3 [v2:172.21.15.156:6826/2037250231,v1:172.21.15.156:6827/2037250231]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:51:30.814 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:30 smithi156 ceph-mon[98743]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:51:30.814 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:30 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3059372379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:30.815 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:30 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:30.815 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:30 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:30.815 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:30 smithi156 ceph-mon[103290]: from='osd.3 [v2:172.21.15.156:6826/2037250231,v1:172.21.15.156:6827/2037250231]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:51:30.815 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:30 smithi156 ceph-mon[103290]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:51:30.815 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:30 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3059372379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:30.815 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:30 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:30.815 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:30 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:30.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:30 smithi171 ceph-mon[90566]: from='osd.3 [v2:172.21.15.156:6826/2037250231,v1:172.21.15.156:6827/2037250231]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:51:30.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:30 smithi171 ceph-mon[90566]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:51:31.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:30 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3059372379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:31.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:30 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:31.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:30 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:31.093 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:51:30 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3[129212]: 2024-04-26T04:51:30.811+0000 7fa2fe850700 -1 osd.3 93 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:51:31.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:31 smithi156 ceph-mon[98743]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:51:31.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:31 smithi156 ceph-mon[98743]: osdmap e96: 8 total, 7 up, 8 in 2024-04-26T04:51:31.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:31 smithi156 ceph-mon[98743]: from='osd.3 [v2:172.21.15.156:6826/2037250231,v1:172.21.15.156:6827/2037250231]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:51:31.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:31 smithi156 ceph-mon[98743]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:51:31.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:31 smithi156 ceph-mon[98743]: pgmap v230: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:31.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:31 smithi156 ceph-mon[98743]: from='osd.3 ' entity='osd.3' 2024-04-26T04:51:31.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:31 smithi156 ceph-mon[103290]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:51:31.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:31 smithi156 ceph-mon[103290]: osdmap e96: 8 total, 7 up, 8 in 2024-04-26T04:51:31.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:31 smithi156 ceph-mon[103290]: from='osd.3 [v2:172.21.15.156:6826/2037250231,v1:172.21.15.156:6827/2037250231]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:51:31.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:31 smithi156 ceph-mon[103290]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:51:31.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:31 smithi156 ceph-mon[103290]: pgmap v230: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:31.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:31 smithi156 ceph-mon[103290]: from='osd.3 ' entity='osd.3' 2024-04-26T04:51:31.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:31 smithi171 ceph-mon[90566]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:51:31.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:31 smithi171 ceph-mon[90566]: osdmap e96: 8 total, 7 up, 8 in 2024-04-26T04:51:31.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:31 smithi171 ceph-mon[90566]: from='osd.3 [v2:172.21.15.156:6826/2037250231,v1:172.21.15.156:6827/2037250231]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:51:32.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:31 smithi171 ceph-mon[90566]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-04-26T04:51:32.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:31 smithi171 ceph-mon[90566]: pgmap v230: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:32.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:31 smithi171 ceph-mon[90566]: from='osd.3 ' entity='osd.3' 2024-04-26T04:51:32.342 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 4 2024-04-26T04:51:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:32 smithi156 ceph-mon[98743]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:51:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:32 smithi156 ceph-mon[98743]: osd.3 [v2:172.21.15.156:6826/2037250231,v1:172.21.15.156:6827/2037250231] boot 2024-04-26T04:51:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:32 smithi156 ceph-mon[98743]: osdmap e97: 8 total, 8 up, 8 in 2024-04-26T04:51:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:32 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:32.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:32 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/4061095589' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:32.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:32.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:32.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:32 smithi156 ceph-mon[103290]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:51:32.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:32 smithi156 ceph-mon[103290]: osd.3 [v2:172.21.15.156:6826/2037250231,v1:172.21.15.156:6827/2037250231] boot 2024-04-26T04:51:32.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:32 smithi156 ceph-mon[103290]: osdmap e97: 8 total, 8 up, 8 in 2024-04-26T04:51:32.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:32 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:32.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:32 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/4061095589' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:32.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:32.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:33.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:32 smithi171 ceph-mon[90566]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:51:33.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:32 smithi171 ceph-mon[90566]: osd.3 [v2:172.21.15.156:6826/2037250231,v1:172.21.15.156:6827/2037250231] boot 2024-04-26T04:51:33.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:32 smithi171 ceph-mon[90566]: osdmap e97: 8 total, 8 up, 8 in 2024-04-26T04:51:33.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:32 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:33.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:32 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/4061095589' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:33.739 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T04:51:34.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:33 smithi156 ceph-mon[98743]: pgmap v232: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:34.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:33 smithi156 ceph-mon[98743]: osdmap e98: 8 total, 8 up, 8 in 2024-04-26T04:51:34.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:33 smithi156 ceph-mon[103290]: pgmap v232: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:34.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:33 smithi156 ceph-mon[103290]: osdmap e98: 8 total, 8 up, 8 in 2024-04-26T04:51:34.242 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:34.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:33 smithi171 ceph-mon[90566]: pgmap v232: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:34.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:33 smithi171 ceph-mon[90566]: osdmap e98: 8 total, 8 up, 8 in 2024-04-26T04:51:34.961 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:51:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:51:34] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.43.0" 2024-04-26T04:51:36.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:35 smithi171 ceph-mon[90566]: pgmap v234: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:36.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:35 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:36.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:35 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:36.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:35 smithi156 ceph-mon[98743]: pgmap v234: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:36.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:36.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:36.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:35 smithi156 ceph-mon[103290]: pgmap v234: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:51:36.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:36.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:37.090 INFO:teuthology.orchestra.run.smithi156.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:37.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:37 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:37 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:37 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:37 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:37 smithi171 ceph-mon[90566]: pgmap v235: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:37 smithi171 ceph-mon[90566]: pgmap v236: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:37 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:37 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:37 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:37 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:37 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:37.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:37 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:51:37.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:37 smithi171 ceph-mon[90566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:51:37.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:37 smithi171 ceph-mon[90566]: Upgrade: osd.4 is safe to restart 2024-04-26T04:51:37.751 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:37 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:37.812 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:37.812 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:37.812 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:37.812 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:37.813 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[103290]: pgmap v235: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:37.813 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[103290]: pgmap v236: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:37.813 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:37.813 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:37.813 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:37.813 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:37.813 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:37.813 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:51:37.813 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[103290]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:51:37.813 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[103290]: Upgrade: osd.4 is safe to restart 2024-04-26T04:51:37.813 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:37.814 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:37.814 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:37.814 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:37.814 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:37.814 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[98743]: pgmap v235: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:37.814 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[98743]: pgmap v236: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:37.814 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:37.814 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:37.814 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:37.814 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:37.815 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:37.815 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:51:37.815 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[98743]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:51:37.815 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[98743]: Upgrade: osd.4 is safe to restart 2024-04-26T04:51:37.815 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:37 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:37.841 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:51:38.102 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:38.499 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:38 smithi156 ceph-mon[103290]: from='client.44504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:38.499 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:38 smithi156 ceph-mon[103290]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:38.499 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:38 smithi156 ceph-mon[103290]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2024-04-26T04:51:38.500 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:38 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:38.500 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:38 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:51:38.500 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:38 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:38.500 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:38 smithi156 ceph-mon[98743]: from='client.44504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:38.500 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:38 smithi156 ceph-mon[98743]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:38.500 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:38 smithi156 ceph-mon[98743]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2024-04-26T04:51:38.500 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:38 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:38.500 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:38 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:51:38.500 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:38 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:38.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:38 smithi171 ceph-mon[90566]: from='client.44504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:38.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:38 smithi171 ceph-mon[90566]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:38.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:38 smithi171 ceph-mon[90566]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2024-04-26T04:51:38.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:38 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:38.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:38 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:51:38.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:38 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:39.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:39 smithi156 ceph-mon[98743]: Upgrade: Updating osd.4 2024-04-26T04:51:39.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:39 smithi156 ceph-mon[98743]: Deploying daemon osd.4 on smithi171 2024-04-26T04:51:39.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:39 smithi156 ceph-mon[98743]: pgmap v237: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:39.711 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:39 smithi156 ceph-mon[103290]: Upgrade: Updating osd.4 2024-04-26T04:51:39.711 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:39 smithi156 ceph-mon[103290]: Deploying daemon osd.4 on smithi171 2024-04-26T04:51:39.711 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:39 smithi156 ceph-mon[103290]: pgmap v237: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:39.735 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:39 smithi171 ceph-mon[90566]: Upgrade: Updating osd.4 2024-04-26T04:51:39.735 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:39 smithi171 ceph-mon[90566]: Deploying daemon osd.4 on smithi171 2024-04-26T04:51:39.735 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:39 smithi171 ceph-mon[90566]: pgmap v237: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:40.160 INFO:teuthology.orchestra.run.smithi156.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:51:40.161 INFO:teuthology.orchestra.run.smithi156.stdout:alertmanager.a smithi156 *:9093,9094 running (7m) 9s ago 11m 24.2M - 0.25.0 c8568f914cd2 932345b7da32 2024-04-26T04:51:40.161 INFO:teuthology.orchestra.run.smithi156.stdout:grafana.a smithi171 *:3000 error 44s ago 11m - - 2024-04-26T04:51:40.161 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi156.wyfzii smithi156 running (10m) 9s ago 10m 17.6M - 17.2.7-902-gc61472c8 0de407c1b3af f42880bb83a3 2024-04-26T04:51:40.161 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi171.kzjlqn smithi171 running (10m) 44s ago 10m 15.2M - 17.2.7-902-gc61472c8 0de407c1b3af 31b518bc187c 2024-04-26T04:51:40.161 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.x smithi171 *:8443,9283,8765 running (7m) 44s ago 15m 451M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 9ec4d348d704 2024-04-26T04:51:40.161 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.y smithi156 *:8443,9283,8765 running (7m) 9s ago 17m 508M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 715bee4dd354 2024-04-26T04:51:40.161 INFO:teuthology.orchestra.run.smithi156.stdout:mon.a smithi156 running (6m) 9s ago 17m 53.9M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b94e4b5539a1 2024-04-26T04:51:40.161 INFO:teuthology.orchestra.run.smithi156.stdout:mon.b smithi171 running (3m) 44s ago 16m 34.8M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0829e2e150f0 2024-04-26T04:51:40.161 INFO:teuthology.orchestra.run.smithi156.stdout:mon.c smithi156 running (5m) 9s ago 16m 38.9M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 8cac6c96e0db 2024-04-26T04:51:40.161 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.a smithi156 *:9100 running (7m) 9s ago 11m 17.8M - 1.5.0 0da6a335fe13 5af6998cc7e3 2024-04-26T04:51:40.161 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.b smithi171 *:9100 running (7m) 44s ago 11m 15.2M - 1.5.0 0da6a335fe13 a2d37cfca8cf 2024-04-26T04:51:40.161 INFO:teuthology.orchestra.run.smithi156.stdout:osd.0 smithi156 running (118s) 9s ago 15m 48.0M 2480M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 7d9e99b5875c 2024-04-26T04:51:40.161 INFO:teuthology.orchestra.run.smithi156.stdout:osd.1 smithi156 running (83s) 9s ago 15m 47.1M 2480M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 3e65e778dd5a 2024-04-26T04:51:40.162 INFO:teuthology.orchestra.run.smithi156.stdout:osd.2 smithi156 running (48s) 9s ago 14m 41.4M 2480M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 fd77f3d69083 2024-04-26T04:51:40.162 INFO:teuthology.orchestra.run.smithi156.stdout:osd.3 smithi156 running (14s) 9s ago 14m 12.6M 2480M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 698081f2c4f0 2024-04-26T04:51:40.162 INFO:teuthology.orchestra.run.smithi156.stdout:osd.4 smithi171 running (14m) 44s ago 14m 52.0M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 388a24a70001 2024-04-26T04:51:40.162 INFO:teuthology.orchestra.run.smithi156.stdout:osd.5 smithi171 running (13m) 44s ago 13m 54.1M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 703235a355b5 2024-04-26T04:51:40.162 INFO:teuthology.orchestra.run.smithi156.stdout:osd.6 smithi171 running (13m) 44s ago 13m 48.7M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 3dad877f8247 2024-04-26T04:51:40.162 INFO:teuthology.orchestra.run.smithi156.stdout:osd.7 smithi171 running (12m) 44s ago 12m 54.3M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e275b2da9b18 2024-04-26T04:51:40.162 INFO:teuthology.orchestra.run.smithi156.stdout:prometheus.a smithi171 *:9095 running (7m) 44s ago 12m 63.8M - 2.43.0 a07b618ecd1d 93d6b22f6fce 2024-04-26T04:51:40.956 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:51:41.220 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:41.622 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:41 smithi156 ceph-mon[103290]: from='client.44510 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:41.622 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:41 smithi156 ceph-mon[103290]: pgmap v238: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:41.623 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:41 smithi156 ceph-mon[98743]: from='client.44510 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:41.623 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:41 smithi156 ceph-mon[98743]: pgmap v238: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:41.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:41 smithi171 ceph-mon[90566]: from='client.44510 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:41.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:41 smithi171 ceph-mon[90566]: pgmap v238: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:41.750 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:41 smithi171 systemd[1]: Stopping Ceph osd.4 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:51:41.750 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:41 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[36606]: 2024-04-26T04:51:41.707+0000 7efe635cd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:51:41.750 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:41 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[36606]: 2024-04-26T04:51:41.707+0000 7efe635cd700 -1 osd.4 98 *** Got signal Terminated *** 2024-04-26T04:51:41.750 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:41 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[36606]: 2024-04-26T04:51:41.707+0000 7efe635cd700 -1 osd.4 98 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:51:42.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:42 smithi171 ceph-mon[90566]: osd.4 marked itself down and dead 2024-04-26T04:51:42.779 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:42 smithi156 ceph-mon[98743]: osd.4 marked itself down and dead 2024-04-26T04:51:42.779 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:42 smithi156 ceph-mon[103290]: osd.4 marked itself down and dead 2024-04-26T04:51:43.206 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:42 smithi171 podman[97123]: 2024-04-26 04:51:42.923315145 +0000 UTC m=+1.309088642 container died 388a24a70001db2d218a67a20c29d3c9b725eb8fbc37ce2ea4bce3284e301a0e (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-26T04:51:43.257 INFO:teuthology.orchestra.run.smithi156.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:51:43.257 INFO:teuthology.orchestra.run.smithi156.stdout:alertmanager.a smithi156 *:9093,9094 running (8m) 12s ago 11m 24.2M - 0.25.0 c8568f914cd2 932345b7da32 2024-04-26T04:51:43.257 INFO:teuthology.orchestra.run.smithi156.stdout:grafana.a smithi171 *:3000 error 47s ago 11m - - 2024-04-26T04:51:43.257 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi156.wyfzii smithi156 running (10m) 12s ago 10m 17.6M - 17.2.7-902-gc61472c8 0de407c1b3af f42880bb83a3 2024-04-26T04:51:43.257 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi171.kzjlqn smithi171 running (10m) 47s ago 10m 15.2M - 17.2.7-902-gc61472c8 0de407c1b3af 31b518bc187c 2024-04-26T04:51:43.257 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.x smithi171 *:8443,9283,8765 running (7m) 47s ago 16m 451M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 9ec4d348d704 2024-04-26T04:51:43.257 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.y smithi156 *:8443,9283,8765 running (7m) 12s ago 17m 508M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 715bee4dd354 2024-04-26T04:51:43.257 INFO:teuthology.orchestra.run.smithi156.stdout:mon.a smithi156 running (6m) 12s ago 17m 53.9M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 b94e4b5539a1 2024-04-26T04:51:43.257 INFO:teuthology.orchestra.run.smithi156.stdout:mon.b smithi171 running (3m) 47s ago 16m 34.8M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 0829e2e150f0 2024-04-26T04:51:43.258 INFO:teuthology.orchestra.run.smithi156.stdout:mon.c smithi156 running (5m) 12s ago 16m 38.9M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 8cac6c96e0db 2024-04-26T04:51:43.258 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.a smithi156 *:9100 running (7m) 12s ago 11m 17.8M - 1.5.0 0da6a335fe13 5af6998cc7e3 2024-04-26T04:51:43.258 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.b smithi171 *:9100 running (7m) 47s ago 11m 15.2M - 1.5.0 0da6a335fe13 a2d37cfca8cf 2024-04-26T04:51:43.258 INFO:teuthology.orchestra.run.smithi156.stdout:osd.0 smithi156 running (2m) 12s ago 15m 48.0M 2480M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 7d9e99b5875c 2024-04-26T04:51:43.258 INFO:teuthology.orchestra.run.smithi156.stdout:osd.1 smithi156 running (86s) 12s ago 15m 47.1M 2480M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 3e65e778dd5a 2024-04-26T04:51:43.258 INFO:teuthology.orchestra.run.smithi156.stdout:osd.2 smithi156 running (51s) 12s ago 14m 41.4M 2480M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 fd77f3d69083 2024-04-26T04:51:43.258 INFO:teuthology.orchestra.run.smithi156.stdout:osd.3 smithi156 running (17s) 12s ago 14m 12.6M 2480M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 698081f2c4f0 2024-04-26T04:51:43.258 INFO:teuthology.orchestra.run.smithi156.stdout:osd.4 smithi171 running (14m) 47s ago 14m 52.0M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 388a24a70001 2024-04-26T04:51:43.258 INFO:teuthology.orchestra.run.smithi156.stdout:osd.5 smithi171 running (13m) 47s ago 13m 54.1M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 703235a355b5 2024-04-26T04:51:43.258 INFO:teuthology.orchestra.run.smithi156.stdout:osd.6 smithi171 running (13m) 47s ago 13m 48.7M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 3dad877f8247 2024-04-26T04:51:43.258 INFO:teuthology.orchestra.run.smithi156.stdout:osd.7 smithi171 running (12m) 47s ago 12m 54.3M 2480M 17.2.7-902-gc61472c8 0de407c1b3af e275b2da9b18 2024-04-26T04:51:43.258 INFO:teuthology.orchestra.run.smithi156.stdout:prometheus.a smithi171 *:9095 running (7m) 47s ago 12m 63.8M - 2.43.0 a07b618ecd1d 93d6b22f6fce 2024-04-26T04:51:43.476 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:43 smithi171 podman[97123]: 2024-04-26 04:51:43.296308736 +0000 UTC m=+1.682082235 container cleanup 388a24a70001db2d218a67a20c29d3c9b725eb8fbc37ce2ea4bce3284e301a0e (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, ceph=True, RELEASE=quincy-c61472c, io.buildah.version=1.33.5) 2024-04-26T04:51:43.476 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:43 smithi171 bash[97123]: ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4 2024-04-26T04:51:43.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:43 smithi171 ceph-mon[90566]: pgmap v239: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:43.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:43 smithi171 ceph-mon[90566]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:51:43.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:43 smithi171 ceph-mon[90566]: osdmap e99: 8 total, 7 up, 8 in 2024-04-26T04:51:43.750 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:43 smithi171 podman[97145]: 2024-04-26 04:51:43.489846586 +0000 UTC m=+0.563004129 container remove 388a24a70001db2d218a67a20c29d3c9b725eb8fbc37ce2ea4bce3284e301a0e (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-26T04:51:43.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:43 smithi156 ceph-mon[98743]: pgmap v239: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:43.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:43 smithi156 ceph-mon[98743]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:51:43.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:43 smithi156 ceph-mon[98743]: osdmap e99: 8 total, 7 up, 8 in 2024-04-26T04:51:43.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:43 smithi156 ceph-mon[103290]: pgmap v239: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:43.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:43 smithi156 ceph-mon[103290]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:51:43.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:43 smithi156 ceph-mon[103290]: osdmap e99: 8 total, 7 up, 8 in 2024-04-26T04:51:43.935 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 110027f2-0386-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions' 2024-04-26T04:51:44.191 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:44.217 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:43 smithi171 podman[97244]: 2024-04-26 04:51:43.796700681 +0000 UTC m=+0.018822935 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:44.218 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:43 smithi171 podman[97244]: 2024-04-26 04:51:43.898740492 +0000 UTC m=+0.120862787 container create 9d694294df05a76889c56423540e43738fdeb34455d3d499a60edd8f3382ba00 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:51:44.218 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:44 smithi171 podman[97244]: 2024-04-26 04:51:44.126139102 +0000 UTC m=+0.348261355 container init 9d694294df05a76889c56423540e43738fdeb34455d3d499a60edd8f3382ba00 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-deactivate, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux ) 2024-04-26T04:51:44.218 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:44 smithi171 podman[97244]: 2024-04-26 04:51:44.131621726 +0000 UTC m=+0.353743982 container start 9d694294df05a76889c56423540e43738fdeb34455d3d499a60edd8f3382ba00 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-26T04:51:44.218 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:44 smithi171 podman[97244]: 2024-04-26 04:51:44.217507061 +0000 UTC m=+0.439629314 container attach 9d694294df05a76889c56423540e43738fdeb34455d3d499a60edd8f3382ba00 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-26T04:51:44.487 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:44 smithi171 podman[97244]: 2024-04-26 04:51:44.423282438 +0000 UTC m=+0.645404694 container died 9d694294df05a76889c56423540e43738fdeb34455d3d499a60edd8f3382ba00 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-26T04:51:44.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:44 smithi156 ceph-mon[98743]: from='client.34614 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:44.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:44 smithi156 ceph-mon[98743]: osdmap e100: 8 total, 7 up, 8 in 2024-04-26T04:51:44.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:44 smithi156 ceph-mon[103290]: from='client.34614 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:44.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:44 smithi156 ceph-mon[103290]: osdmap e100: 8 total, 7 up, 8 in 2024-04-26T04:51:44.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:44 smithi171 ceph-mon[90566]: from='client.34614 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:44.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:44 smithi171 ceph-mon[90566]: osdmap e100: 8 total, 7 up, 8 in 2024-04-26T04:51:45.083 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:51:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:51:44] "GET /metrics HTTP/1.1" 200 33750 "" "Prometheus/2.43.0" 2024-04-26T04:51:45.478 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:45 smithi171 podman[97290]: 2024-04-26 04:51:45.021013013 +0000 UTC m=+0.589446747 container remove 9d694294df05a76889c56423540e43738fdeb34455d3d499a60edd8f3382ba00 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True) 2024-04-26T04:51:45.478 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:45 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.4.service: Deactivated successfully. 2024-04-26T04:51:45.478 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:45 smithi171 systemd[1]: Stopped Ceph osd.4 for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:51:45.478 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:45 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.4.service: Consumed 6.212s CPU time. 2024-04-26T04:51:45.479 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:45 smithi171 systemd[1]: Starting Ceph osd.4 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:51:45.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:45 smithi171 ceph-mon[90566]: pgmap v242: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:45.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:45 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:45.750 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:45 smithi171 podman[97403]: 2024-04-26 04:51:45.380573209 +0000 UTC m=+0.018084179 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:45.750 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:45 smithi171 podman[97403]: 2024-04-26 04:51:45.514113232 +0000 UTC m=+0.151624203 container create 0a7911bac997b29d241190be4fbef4cda05c40847ddec180b278061bb78e1f7d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-04-26T04:51:45.820 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:45 smithi156 ceph-mon[98743]: pgmap v242: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:45.820 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:45 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:45.820 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:45 smithi156 ceph-mon[103290]: pgmap v242: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:45.820 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:45 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:46.207 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:45 smithi171 podman[97403]: 2024-04-26 04:51:45.758392483 +0000 UTC m=+0.395903472 container init 0a7911bac997b29d241190be4fbef4cda05c40847ddec180b278061bb78e1f7d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-26T04:51:46.208 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:45 smithi171 podman[97403]: 2024-04-26 04:51:45.763900522 +0000 UTC m=+0.401411493 container start 0a7911bac997b29d241190be4fbef4cda05c40847ddec180b278061bb78e1f7d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:51:46.208 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:45 smithi171 podman[97403]: 2024-04-26 04:51:45.858194911 +0000 UTC m=+0.495705899 container attach 0a7911bac997b29d241190be4fbef4cda05c40847ddec180b278061bb78e1f7d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-26T04:51:46.345 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2024-04-26T04:51:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: "mon": { 2024-04-26T04:51:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-26T04:51:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2024-04-26T04:51:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: "mgr": { 2024-04-26T04:51:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-26T04:51:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2024-04-26T04:51:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: "osd": { 2024-04-26T04:51:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)": 3, 2024-04-26T04:51:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 4 2024-04-26T04:51:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2024-04-26T04:51:46.346 INFO:teuthology.orchestra.run.smithi156.stdout: "mds": { 2024-04-26T04:51:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)": 2 2024-04-26T04:51:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2024-04-26T04:51:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: "overall": { 2024-04-26T04:51:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)": 5, 2024-04-26T04:51:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 9 2024-04-26T04:51:46.347 INFO:teuthology.orchestra.run.smithi156.stdout: } 2024-04-26T04:51:46.347 INFO:teuthology.orchestra.run.smithi156.stdout:} 2024-04-26T04:51:46.493 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:46 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-activate[97418]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-26T04:51:46.493 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:46 smithi171 bash[97403]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-26T04:51:46.494 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:46 smithi171 bash[97403]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T04:51:46.494 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:46 smithi171 bash[97403]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T04:51:46.494 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:46 smithi171 bash[97403]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T04:51:46.494 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:46 smithi171 bash[97403]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-26T04:51:46.494 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:46 smithi171 bash[97403]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-26T04:51:46.494 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:46 smithi171 bash[97403]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-26T04:51:46.494 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:46 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-activate[97418]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T04:51:46.494 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:46 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-activate[97418]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T04:51:46.494 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:46 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-activate[97418]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T04:51:46.494 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:46 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-activate[97418]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-26T04:51:46.494 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:46 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-activate[97418]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-26T04:51:46.494 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:46 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-activate[97418]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-26T04:51:46.494 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:46 smithi171 podman[97403]: 2024-04-26 04:51:46.23150961 +0000 UTC m=+0.869020580 container died 0a7911bac997b29d241190be4fbef4cda05c40847ddec180b278061bb78e1f7d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-activate, RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-26T04:51:46.649 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:46 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2067165747' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:46.649 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:46 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2067165747' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:46.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:46 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2067165747' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:47.066 DEBUG:teuthology.parallel:result is None 2024-04-26T04:51:47.067 INFO:teuthology.run_tasks:Running task print... 2024-04-26T04:51:47.076 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-04-26T04:51:47.077 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-26T04:51:47.114 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-26T04:51:47.114 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-26T04:51:47.114 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-26T04:51:47.115 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:46 smithi171 podman[97568]: 2024-04-26 04:51:46.811676381 +0000 UTC m=+0.574545653 container remove 0a7911bac997b29d241190be4fbef4cda05c40847ddec180b278061bb78e1f7d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:51:47.115 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:47 smithi171 podman[97638]: 2024-04-26 04:51:46.926602258 +0000 UTC m=+0.018483401 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:47.115 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:47 smithi171 podman[97638]: 2024-04-26 04:51:47.087698316 +0000 UTC m=+0.179579458 container create 9b3702c8c233908df777ca1736d4e6b713e47bbf87dc2b74525544e5c593f151 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:51:47.367 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:47.415 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:47 smithi171 podman[97638]: 2024-04-26 04:51:47.298481252 +0000 UTC m=+0.390362395 container init 9b3702c8c233908df777ca1736d4e6b713e47bbf87dc2b74525544e5c593f151 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True) 2024-04-26T04:51:47.415 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:47 smithi171 podman[97638]: 2024-04-26 04:51:47.304020543 +0000 UTC m=+0.395901686 container start 9b3702c8c233908df777ca1736d4e6b713e47bbf87dc2b74525544e5c593f151 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-26T04:51:47.415 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:47 smithi171 ceph-osd[97676]: -- 172.21.15.171:0/2713568105 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f0d006f680 con 0x55f0d0039400 2024-04-26T04:51:47.749 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:47 smithi171 bash[97638]: 9b3702c8c233908df777ca1736d4e6b713e47bbf87dc2b74525544e5c593f151 2024-04-26T04:51:47.749 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:47 smithi171 systemd[1]: Started Ceph osd.4 for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:51:47.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:47 smithi171 ceph-mon[90566]: pgmap v243: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-04-26T04:51:47.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:47 smithi171 ceph-mon[90566]: Health check failed: Degraded data redundancy: 14/72 objects degraded (19.444%), 9 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:47.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:47 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:47.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:47 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:47.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:47 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:47.787 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:47 smithi156 ceph-mon[98743]: pgmap v243: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-04-26T04:51:47.787 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:47 smithi156 ceph-mon[98743]: Health check failed: Degraded data redundancy: 14/72 objects degraded (19.444%), 9 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:47.787 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:47 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:47.787 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:47 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:47.787 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:47 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:47.787 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:47 smithi156 ceph-mon[103290]: pgmap v243: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-04-26T04:51:47.788 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:47 smithi156 ceph-mon[103290]: Health check failed: Degraded data redundancy: 14/72 objects degraded (19.444%), 9 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:47.788 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:47 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:47.788 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:47 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:47.788 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:47 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:48.249 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:47 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[97653]: 2024-04-26T04:51:47.850+0000 7f7fb727f7c0 -1 Falling back to public interface 2024-04-26T04:51:49.575 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:51:49.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:49 smithi171 ceph-mon[90566]: pgmap v244: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-04-26T04:51:49.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:49 smithi156 ceph-mon[98743]: pgmap v244: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-04-26T04:51:49.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:49 smithi156 ceph-mon[103290]: pgmap v244: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-04-26T04:51:50.232 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":39,"flags":0,"active_gid":25250,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6800","nonce":771638941},{"type":"v1","addr":"172.21.15.156:6801","nonce":771638941}]},"active_addr":"172.21.15.156:6801/771638941","active_change":"2024-04-26T04:45:04.876168+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":34107,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.156:8443/","prometheus":"http://172.21.15.156:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":78,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":2731456651}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":3158599381}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":2333057851}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":2531043021}]}]} 2024-04-26T04:51:50.236 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-26T04:51:50.236 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-26T04:51:50.236 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:51:50.490 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:50.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:50 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2392688374' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:51:50.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:50 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:50.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:50 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:50.750 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:50 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[97653]: 2024-04-26T04:51:50.590+0000 7f7fb727f7c0 -1 osd.4 98 log_to_monitors true 2024-04-26T04:51:50.764 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:50 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2392688374' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:51:50.764 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:50 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:50.764 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:50 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:50.764 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:50 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2392688374' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:51:50.764 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:50 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:50.764 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:50 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:51.590 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:51 smithi156 ceph-mon[103290]: pgmap v245: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-04-26T04:51:51.590 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:51 smithi156 ceph-mon[103290]: from='osd.4 [v2:172.21.15.171:6800/1502285480,v1:172.21.15.171:6801/1502285480]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:51:51.590 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:51 smithi156 ceph-mon[103290]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:51:51.590 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:51 smithi156 ceph-mon[98743]: pgmap v245: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-04-26T04:51:51.590 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:51 smithi156 ceph-mon[98743]: from='osd.4 [v2:172.21.15.171:6800/1502285480,v1:172.21.15.171:6801/1502285480]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:51:51.590 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:51 smithi156 ceph-mon[98743]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:51:51.968 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:51 smithi171 ceph-mon[90566]: pgmap v245: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-04-26T04:51:51.969 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:51 smithi171 ceph-mon[90566]: from='osd.4 [v2:172.21.15.171:6800/1502285480,v1:172.21.15.171:6801/1502285480]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:51:51.969 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:51 smithi171 ceph-mon[90566]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:51:52.249 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:51:51 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[97653]: 2024-04-26T04:51:51.966+0000 7f7fa46cf700 -1 osd.4 98 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:51:52.292 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:51:52.292 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":101,"fsid":"110027f2-0386-11ef-bc93-c7b262605968","created":"2024-04-26T04:33:46.638460+0000","modified":"2024-04-26T04:51:51.494874+0000","last_up_change":"2024-04-26T04:51:42.473764+0000","last_in_change":"2024-04-26T04:38:39.401044+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:36:55.213219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:39:28.969228+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.9900000095367432,"score_stable":1.9900000095367432,"optimal_score":0.87999999523162842,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:40:42.534967+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.7400000095367432,"score_stable":1.7400000095367432,"optimal_score":0.87999999523162842,"raw_score_acting":1.5299999713897705,"raw_score_stable":1.5299999713897705,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:40:43.505179+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.4900000095367432,"score_stable":2.4900000095367432,"optimal_score":0.87999999523162842,"raw_score_acting":2.190000057220459,"raw_score_stable":2.190000057220459,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6b59fe49-0273-4115-91d9-6faea7d37616","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":78,"up_from":82,"up_thru":99,"down_at":79,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":1734867425},{"type":"v1","addr":"172.21.15.156:6803","nonce":1734867425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":1734867425},{"type":"v1","addr":"172.21.15.156:6805","nonce":1734867425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":1734867425},{"type":"v1","addr":"172.21.15.156:6809","nonce":1734867425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":1734867425},{"type":"v1","addr":"172.21.15.156:6807","nonce":1734867425}]},"public_addr":"172.21.15.156:6803/1734867425","cluster_addr":"172.21.15.156:6805/1734867425","heartbeat_back_addr":"172.21.15.156:6809/1734867425","heartbeat_front_addr":"172.21.15.156:6807/1734867425","state":["exists","up"]},{"osd":1,"uuid":"c9673007-264f-4d7c-8234-bfd514e70d2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":83,"up_from":87,"up_thru":99,"down_at":84,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":915951660},{"type":"v1","addr":"172.21.15.156:6811","nonce":915951660}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":915951660},{"type":"v1","addr":"172.21.15.156:6813","nonce":915951660}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":915951660},{"type":"v1","addr":"172.21.15.156:6817","nonce":915951660}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":915951660},{"type":"v1","addr":"172.21.15.156:6815","nonce":915951660}]},"public_addr":"172.21.15.156:6811/915951660","cluster_addr":"172.21.15.156:6813/915951660","heartbeat_back_addr":"172.21.15.156:6817/915951660","heartbeat_front_addr":"172.21.15.156:6815/915951660","state":["exists","up"]},{"osd":2,"uuid":"d0051bbe-c0cd-42f4-9d8c-6b0a3e71886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":88,"up_from":92,"up_thru":99,"down_at":89,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":3864636890},{"type":"v1","addr":"172.21.15.156:6819","nonce":3864636890}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":3864636890},{"type":"v1","addr":"172.21.15.156:6821","nonce":3864636890}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":3864636890},{"type":"v1","addr":"172.21.15.156:6825","nonce":3864636890}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":3864636890},{"type":"v1","addr":"172.21.15.156:6823","nonce":3864636890}]},"public_addr":"172.21.15.156:6819/3864636890","cluster_addr":"172.21.15.156:6821/3864636890","heartbeat_back_addr":"172.21.15.156:6825/3864636890","heartbeat_front_addr":"172.21.15.156:6823/3864636890","state":["exists","up"]},{"osd":3,"uuid":"479e109c-1e4a-4124-9442-f85d63df6d04","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":93,"up_from":97,"up_thru":99,"down_at":94,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2037250231},{"type":"v1","addr":"172.21.15.156:6827","nonce":2037250231}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2037250231},{"type":"v1","addr":"172.21.15.156:6829","nonce":2037250231}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6832","nonce":2037250231},{"type":"v1","addr":"172.21.15.156:6833","nonce":2037250231}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2037250231},{"type":"v1","addr":"172.21.15.156:6831","nonce":2037250231}]},"public_addr":"172.21.15.156:6827/2037250231","cluster_addr":"172.21.15.156:6829/2037250231","heartbeat_back_addr":"172.21.15.156:6833/2037250231","heartbeat_front_addr":"172.21.15.156:6831/2037250231","state":["exists","up"]},{"osd":4,"uuid":"b090dbe4-ed23-4015-a864-074aa64cace4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":97,"down_at":99,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6801","nonce":2255415442}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6803","nonce":2255415442}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6807","nonce":2255415442}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":2255415442},{"type":"v1","addr":"172.21.15.171:6805","nonce":2255415442}]},"public_addr":"172.21.15.171:6801/2255415442","cluster_addr":"172.21.15.171:6803/2255415442","heartbeat_back_addr":"172.21.15.171:6807/2255415442","heartbeat_front_addr":"172.21.15.171:6805/2255415442","state":["exists"]},{"osd":5,"uuid":"f506481f-428b-40d6-a54b-8ef411672b53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6809","nonce":2373798416}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6811","nonce":2373798416}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6815","nonce":2373798416}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6813","nonce":2373798416}]},"public_addr":"172.21.15.171:6809/2373798416","cluster_addr":"172.21.15.171:6811/2373798416","heartbeat_back_addr":"172.21.15.171:6815/2373798416","heartbeat_front_addr":"172.21.15.171:6813/2373798416","state":["exists","up"]},{"osd":6,"uuid":"f1fd019a-372d-4b4e-bede-7659038df10a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6817","nonce":2328644410}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6819","nonce":2328644410}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6823","nonce":2328644410}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6821","nonce":2328644410}]},"public_addr":"172.21.15.171:6817/2328644410","cluster_addr":"172.21.15.171:6819/2328644410","heartbeat_back_addr":"172.21.15.171:6823/2328644410","heartbeat_front_addr":"172.21.15.171:6821/2328644410","state":["exists","up"]},{"osd":7,"uuid":"bee1398e-7912-458a-9f8b-279b84a22566","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6825","nonce":2507226151}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6827","nonce":2507226151}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6831","nonce":2507226151}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6829","nonce":2507226151}]},"public_addr":"172.21.15.171:6825/2507226151","cluster_addr":"172.21.15.171:6827/2507226151","heartbeat_back_addr":"172.21.15.171:6831/2507226151","heartbeat_front_addr":"172.21.15.171:6829/2507226151","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:03.400774+0000","dead_epoch":78},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:28.117542+0000","dead_epoch":83},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:52.563066+0000","dead_epoch":88},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:17.387470+0000","dead_epoch":93},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:40.460850+0000","dead_epoch":98},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:05.110287+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:29.602919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:55.143996+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.e","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:0/16376181":"2024-04-27T04:45:04.875889+0000","172.21.15.171:0/86196127":"2024-04-27T04:44:18.308381+0000","172.21.15.156:0/3331144037":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/402721227":"2024-04-27T04:34:42.908359+0000","172.21.15.171:0/4249015634":"2024-04-27T04:44:18.308381+0000","172.21.15.171:6835/2811112754":"2024-04-27T04:44:18.308381+0000","172.21.15.156:0/532521842":"2024-04-27T04:45:04.875889+0000","172.21.15.156:6801/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.171:6834/2811112754":"2024-04-27T04:44:18.308381+0000","172.21.15.171:0/4114228739":"2024-04-27T04:44:18.308381+0000","172.21.15.156:0/1323755296":"2024-04-27T04:45:04.875889+0000","172.21.15.156:0/3755168891":"2024-04-27T04:39:29.182492+0000","172.21.15.171:0/847470859":"2024-04-27T04:44:18.308381+0000","172.21.15.156:0/4047047180":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/1144243691":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/4143435073":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/711432262":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/67599134":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/2398977550":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6801/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/1361591211":"2024-04-27T04:43:10.958063+0000","172.21.15.156:0/1375023159":"2024-04-27T04:45:04.875889+0000","172.21.15.156:6800/367046062":"2024-04-27T04:43:10.958063+0000","172.21.15.156:6800/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:6800/1084840701":"2024-04-27T04:39:29.182492+0000","172.21.15.156:6800/3726283619":"2024-04-27T04:45:04.875889+0000","172.21.15.156:6801/367046062":"2024-04-27T04:43:10.958063+0000","172.21.15.171:0/2399344582":"2024-04-27T04:44:18.308381+0000","172.21.15.156:0/314056408":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/32257782":"2024-04-27T04:43:10.958063+0000","172.21.15.156:0/3291954384":"2024-04-27T04:43:10.958063+0000","172.21.15.171:0/1847605205":"2024-04-27T04:44:18.308381+0000","172.21.15.156:6800/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6801/1084840701":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/1772897570":"2024-04-27T04:43:10.958063+0000","172.21.15.156:0/2324075789":"2024-04-27T04:45:04.875889+0000","172.21.15.156:0/953980496":"2024-04-27T04:45:04.875889+0000","172.21.15.156:6801/3726283619":"2024-04-27T04:45:04.875889+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:51:52.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:52 smithi156 ceph-mon[103290]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:51:52.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:52 smithi156 ceph-mon[103290]: osdmap e101: 8 total, 7 up, 8 in 2024-04-26T04:51:52.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:52 smithi156 ceph-mon[103290]: from='osd.4 [v2:172.21.15.171:6800/1502285480,v1:172.21.15.171:6801/1502285480]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:51:52.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:52 smithi156 ceph-mon[103290]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:51:52.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:52 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:52.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:52 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:52.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:52 smithi156 ceph-mon[103290]: from='osd.4 ' entity='osd.4' 2024-04-26T04:51:52.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:52 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2210569610' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:51:52.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:52 smithi156 ceph-mon[98743]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:51:52.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:52 smithi156 ceph-mon[98743]: osdmap e101: 8 total, 7 up, 8 in 2024-04-26T04:51:52.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:52 smithi156 ceph-mon[98743]: from='osd.4 [v2:172.21.15.171:6800/1502285480,v1:172.21.15.171:6801/1502285480]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:51:52.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:52 smithi156 ceph-mon[98743]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:51:52.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:52 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:52.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:52 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:52.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:52 smithi156 ceph-mon[98743]: from='osd.4 ' entity='osd.4' 2024-04-26T04:51:52.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:52 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2210569610' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:51:52.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:52 smithi171 ceph-mon[90566]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:51:52.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:52 smithi171 ceph-mon[90566]: osdmap e101: 8 total, 7 up, 8 in 2024-04-26T04:51:52.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:52 smithi171 ceph-mon[90566]: from='osd.4 [v2:172.21.15.171:6800/1502285480,v1:172.21.15.171:6801/1502285480]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:51:53.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:52 smithi171 ceph-mon[90566]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-04-26T04:51:53.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:52 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:53.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:52 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:53.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:52 smithi171 ceph-mon[90566]: from='osd.4 ' entity='osd.4' 2024-04-26T04:51:53.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:52 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2210569610' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:51:53.942 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:53 smithi156 ceph-mon[103290]: pgmap v247: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-04-26T04:51:53.943 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:53 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:53.943 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:53 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:53.943 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:53 smithi156 ceph-mon[103290]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:51:53.943 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:53 smithi156 ceph-mon[103290]: osd.4 [v2:172.21.15.171:6800/1502285480,v1:172.21.15.171:6801/1502285480] boot 2024-04-26T04:51:53.943 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:53 smithi156 ceph-mon[103290]: osdmap e102: 8 total, 8 up, 8 in 2024-04-26T04:51:53.943 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:53 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:51:53.943 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:53 smithi156 ceph-mon[98743]: pgmap v247: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-04-26T04:51:53.943 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:53 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:53.944 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:53 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:53.944 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:53 smithi156 ceph-mon[98743]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:51:53.944 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:53 smithi156 ceph-mon[98743]: osd.4 [v2:172.21.15.171:6800/1502285480,v1:172.21.15.171:6801/1502285480] boot 2024-04-26T04:51:53.944 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:53 smithi156 ceph-mon[98743]: osdmap e102: 8 total, 8 up, 8 in 2024-04-26T04:51:53.944 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:53 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:51:53.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:53 smithi171 ceph-mon[90566]: pgmap v247: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-04-26T04:51:54.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:53 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:54.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:53 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:54.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:53 smithi171 ceph-mon[90566]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:51:54.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:53 smithi171 ceph-mon[90566]: osd.4 [v2:172.21.15.171:6800/1502285480,v1:172.21.15.171:6801/1502285480] boot 2024-04-26T04:51:54.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:53 smithi171 ceph-mon[90566]: osdmap e102: 8 total, 8 up, 8 in 2024-04-26T04:51:54.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:53 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:51:54.969 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:51:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:51:54] "GET /metrics HTTP/1.1" 200 33900 "" "Prometheus/2.43.0" 2024-04-26T04:51:55.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:54 smithi171 ceph-mon[90566]: osdmap e103: 8 total, 8 up, 8 in 2024-04-26T04:51:55.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:54 smithi156 ceph-mon[98743]: osdmap e103: 8 total, 8 up, 8 in 2024-04-26T04:51:55.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:54 smithi156 ceph-mon[103290]: osdmap e103: 8 total, 8 up, 8 in 2024-04-26T04:51:55.996 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:51:56.105 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:55 smithi171 ceph-mon[90566]: pgmap v250: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-04-26T04:51:56.267 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:56.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:55 smithi156 ceph-mon[98743]: pgmap v250: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-04-26T04:51:56.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:55 smithi156 ceph-mon[103290]: pgmap v250: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-04-26T04:51:57.414 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:57.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:57.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:57.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:57.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[103290]: pgmap v251: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-26T04:51:57.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[103290]: pgmap v252: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-26T04:51:57.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:57.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:57.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:57.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:57.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[103290]: pgmap v253: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-26T04:51:57.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[103290]: pgmap v254: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-26T04:51:57.415 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:57.416 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:57.416 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:57.416 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:57.416 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:57.416 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[98743]: pgmap v251: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-26T04:51:57.416 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[98743]: pgmap v252: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-26T04:51:57.416 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:57.416 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:57.416 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:57.416 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:57.417 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[98743]: pgmap v253: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-26T04:51:57.417 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[98743]: pgmap v254: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-26T04:51:57.417 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:57 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:57.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:57 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:57 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:57 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:57 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:57 smithi171 ceph-mon[90566]: pgmap v251: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-26T04:51:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:57 smithi171 ceph-mon[90566]: pgmap v252: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-26T04:51:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:57 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:57 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:57 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:57 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:57 smithi171 ceph-mon[90566]: pgmap v253: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-26T04:51:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:57 smithi171 ceph-mon[90566]: pgmap v254: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2024-04-26T04:51:57.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:57 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:51:58.124 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:51:58.125 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":103,"fsid":"110027f2-0386-11ef-bc93-c7b262605968","created":"2024-04-26T04:33:46.638460+0000","modified":"2024-04-26T04:51:53.961742+0000","last_up_change":"2024-04-26T04:51:52.955077+0000","last_in_change":"2024-04-26T04:38:39.401044+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:36:55.213219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:39:28.969228+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:40:42.534967+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:40:43.505179+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6b59fe49-0273-4115-91d9-6faea7d37616","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":78,"up_from":82,"up_thru":102,"down_at":79,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":1734867425},{"type":"v1","addr":"172.21.15.156:6803","nonce":1734867425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":1734867425},{"type":"v1","addr":"172.21.15.156:6805","nonce":1734867425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":1734867425},{"type":"v1","addr":"172.21.15.156:6809","nonce":1734867425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":1734867425},{"type":"v1","addr":"172.21.15.156:6807","nonce":1734867425}]},"public_addr":"172.21.15.156:6803/1734867425","cluster_addr":"172.21.15.156:6805/1734867425","heartbeat_back_addr":"172.21.15.156:6809/1734867425","heartbeat_front_addr":"172.21.15.156:6807/1734867425","state":["exists","up"]},{"osd":1,"uuid":"c9673007-264f-4d7c-8234-bfd514e70d2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":83,"up_from":87,"up_thru":102,"down_at":84,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":915951660},{"type":"v1","addr":"172.21.15.156:6811","nonce":915951660}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":915951660},{"type":"v1","addr":"172.21.15.156:6813","nonce":915951660}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":915951660},{"type":"v1","addr":"172.21.15.156:6817","nonce":915951660}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":915951660},{"type":"v1","addr":"172.21.15.156:6815","nonce":915951660}]},"public_addr":"172.21.15.156:6811/915951660","cluster_addr":"172.21.15.156:6813/915951660","heartbeat_back_addr":"172.21.15.156:6817/915951660","heartbeat_front_addr":"172.21.15.156:6815/915951660","state":["exists","up"]},{"osd":2,"uuid":"d0051bbe-c0cd-42f4-9d8c-6b0a3e71886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":88,"up_from":92,"up_thru":102,"down_at":89,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":3864636890},{"type":"v1","addr":"172.21.15.156:6819","nonce":3864636890}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":3864636890},{"type":"v1","addr":"172.21.15.156:6821","nonce":3864636890}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":3864636890},{"type":"v1","addr":"172.21.15.156:6825","nonce":3864636890}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":3864636890},{"type":"v1","addr":"172.21.15.156:6823","nonce":3864636890}]},"public_addr":"172.21.15.156:6819/3864636890","cluster_addr":"172.21.15.156:6821/3864636890","heartbeat_back_addr":"172.21.15.156:6825/3864636890","heartbeat_front_addr":"172.21.15.156:6823/3864636890","state":["exists","up"]},{"osd":3,"uuid":"479e109c-1e4a-4124-9442-f85d63df6d04","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":93,"up_from":97,"up_thru":102,"down_at":94,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2037250231},{"type":"v1","addr":"172.21.15.156:6827","nonce":2037250231}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2037250231},{"type":"v1","addr":"172.21.15.156:6829","nonce":2037250231}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6832","nonce":2037250231},{"type":"v1","addr":"172.21.15.156:6833","nonce":2037250231}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2037250231},{"type":"v1","addr":"172.21.15.156:6831","nonce":2037250231}]},"public_addr":"172.21.15.156:6827/2037250231","cluster_addr":"172.21.15.156:6829/2037250231","heartbeat_back_addr":"172.21.15.156:6833/2037250231","heartbeat_front_addr":"172.21.15.156:6831/2037250231","state":["exists","up"]},{"osd":4,"uuid":"b090dbe4-ed23-4015-a864-074aa64cace4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":98,"up_from":102,"up_thru":102,"down_at":99,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":1502285480},{"type":"v1","addr":"172.21.15.171:6801","nonce":1502285480}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":1502285480},{"type":"v1","addr":"172.21.15.171:6803","nonce":1502285480}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":1502285480},{"type":"v1","addr":"172.21.15.171:6807","nonce":1502285480}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":1502285480},{"type":"v1","addr":"172.21.15.171:6805","nonce":1502285480}]},"public_addr":"172.21.15.171:6801/1502285480","cluster_addr":"172.21.15.171:6803/1502285480","heartbeat_back_addr":"172.21.15.171:6807/1502285480","heartbeat_front_addr":"172.21.15.171:6805/1502285480","state":["exists","up"]},{"osd":5,"uuid":"f506481f-428b-40d6-a54b-8ef411672b53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":102,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6809","nonce":2373798416}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6811","nonce":2373798416}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6815","nonce":2373798416}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6813","nonce":2373798416}]},"public_addr":"172.21.15.171:6809/2373798416","cluster_addr":"172.21.15.171:6811/2373798416","heartbeat_back_addr":"172.21.15.171:6815/2373798416","heartbeat_front_addr":"172.21.15.171:6813/2373798416","state":["exists","up"]},{"osd":6,"uuid":"f1fd019a-372d-4b4e-bede-7659038df10a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":102,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6817","nonce":2328644410}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6819","nonce":2328644410}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6823","nonce":2328644410}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6821","nonce":2328644410}]},"public_addr":"172.21.15.171:6817/2328644410","cluster_addr":"172.21.15.171:6819/2328644410","heartbeat_back_addr":"172.21.15.171:6823/2328644410","heartbeat_front_addr":"172.21.15.171:6821/2328644410","state":["exists","up"]},{"osd":7,"uuid":"bee1398e-7912-458a-9f8b-279b84a22566","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":102,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6825","nonce":2507226151}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6827","nonce":2507226151}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6831","nonce":2507226151}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6829","nonce":2507226151}]},"public_addr":"172.21.15.171:6825/2507226151","cluster_addr":"172.21.15.171:6827/2507226151","heartbeat_back_addr":"172.21.15.171:6831/2507226151","heartbeat_front_addr":"172.21.15.171:6829/2507226151","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:03.400774+0000","dead_epoch":78},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:28.117542+0000","dead_epoch":83},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:52.563066+0000","dead_epoch":88},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:17.387470+0000","dead_epoch":93},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:40.460850+0000","dead_epoch":98},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:05.110287+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:29.602919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:55.143996+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.e","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:0/953980496":"2024-04-27T04:45:04.875889+0000","172.21.15.156:0/1375023159":"2024-04-27T04:45:04.875889+0000","172.21.15.156:0/1361591211":"2024-04-27T04:43:10.958063+0000","172.21.15.171:6835/2811112754":"2024-04-27T04:44:18.308381+0000","172.21.15.171:6834/2811112754":"2024-04-27T04:44:18.308381+0000","172.21.15.171:0/4114228739":"2024-04-27T04:44:18.308381+0000","172.21.15.156:6801/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:6801/1084840701":"2024-04-27T04:39:29.182492+0000","172.21.15.156:6800/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/1323755296":"2024-04-27T04:45:04.875889+0000","172.21.15.156:0/1772897570":"2024-04-27T04:43:10.958063+0000","172.21.15.156:0/2324075789":"2024-04-27T04:45:04.875889+0000","172.21.15.156:0/16376181":"2024-04-27T04:45:04.875889+0000","172.21.15.156:0/711432262":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/67599134":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/2398977550":"2024-04-27T04:34:18.740320+0000","172.21.15.171:0/4249015634":"2024-04-27T04:44:18.308381+0000","172.21.15.156:6801/3726283619":"2024-04-27T04:45:04.875889+0000","172.21.15.156:0/3331144037":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/402721227":"2024-04-27T04:34:42.908359+0000","172.21.15.156:6801/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/532521842":"2024-04-27T04:45:04.875889+0000","172.21.15.171:0/847470859":"2024-04-27T04:44:18.308381+0000","172.21.15.156:0/4047047180":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/3755168891":"2024-04-27T04:39:29.182492+0000","172.21.15.171:0/86196127":"2024-04-27T04:44:18.308381+0000","172.21.15.156:0/4143435073":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/1144243691":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6800/367046062":"2024-04-27T04:43:10.958063+0000","172.21.15.156:6800/1084840701":"2024-04-27T04:39:29.182492+0000","172.21.15.156:6800/3726283619":"2024-04-27T04:45:04.875889+0000","172.21.15.156:6800/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:6801/367046062":"2024-04-27T04:43:10.958063+0000","172.21.15.171:0/2399344582":"2024-04-27T04:44:18.308381+0000","172.21.15.156:0/314056408":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/32257782":"2024-04-27T04:43:10.958063+0000","172.21.15.156:0/3291954384":"2024-04-27T04:43:10.958063+0000","172.21.15.171:0/1847605205":"2024-04-27T04:44:18.308381+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:51:58.421 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:58 smithi156 ceph-mon[98743]: Health check update: Degraded data redundancy: 5/72 objects degraded (6.944%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:58.421 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:58 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/438663591' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:51:58.421 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:58 smithi156 ceph-mon[103290]: Health check update: Degraded data redundancy: 5/72 objects degraded (6.944%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:58.422 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:58 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/438663591' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:51:58.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:58 smithi171 ceph-mon[90566]: Health check update: Degraded data redundancy: 5/72 objects degraded (6.944%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:58.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:58 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/438663591' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:51:58.754 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-26T04:51:58.755 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:51:59.005 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:59.433 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:51:59 smithi156 ceph-mon[98743]: pgmap v255: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:59.433 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:51:59 smithi156 ceph-mon[103290]: pgmap v255: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:51:59.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:51:59 smithi171 ceph-mon[90566]: pgmap v255: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:00.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:00 smithi171 ceph-mon[90566]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/72 objects degraded (6.944%), 2 pgs degraded) 2024-04-26T04:52:00.530 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:00 smithi156 ceph-mon[98743]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/72 objects degraded (6.944%), 2 pgs degraded) 2024-04-26T04:52:00.531 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:00 smithi156 ceph-mon[103290]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/72 objects degraded (6.944%), 2 pgs degraded) 2024-04-26T04:52:00.973 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:52:00.973 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":103,"fsid":"110027f2-0386-11ef-bc93-c7b262605968","created":"2024-04-26T04:33:46.638460+0000","modified":"2024-04-26T04:51:53.961742+0000","last_up_change":"2024-04-26T04:51:52.955077+0000","last_in_change":"2024-04-26T04:38:39.401044+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:36:55.213219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:39:28.969228+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:40:42.534967+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:40:43.505179+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6b59fe49-0273-4115-91d9-6faea7d37616","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":78,"up_from":82,"up_thru":102,"down_at":79,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":1734867425},{"type":"v1","addr":"172.21.15.156:6803","nonce":1734867425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":1734867425},{"type":"v1","addr":"172.21.15.156:6805","nonce":1734867425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":1734867425},{"type":"v1","addr":"172.21.15.156:6809","nonce":1734867425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":1734867425},{"type":"v1","addr":"172.21.15.156:6807","nonce":1734867425}]},"public_addr":"172.21.15.156:6803/1734867425","cluster_addr":"172.21.15.156:6805/1734867425","heartbeat_back_addr":"172.21.15.156:6809/1734867425","heartbeat_front_addr":"172.21.15.156:6807/1734867425","state":["exists","up"]},{"osd":1,"uuid":"c9673007-264f-4d7c-8234-bfd514e70d2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":83,"up_from":87,"up_thru":102,"down_at":84,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":915951660},{"type":"v1","addr":"172.21.15.156:6811","nonce":915951660}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":915951660},{"type":"v1","addr":"172.21.15.156:6813","nonce":915951660}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":915951660},{"type":"v1","addr":"172.21.15.156:6817","nonce":915951660}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":915951660},{"type":"v1","addr":"172.21.15.156:6815","nonce":915951660}]},"public_addr":"172.21.15.156:6811/915951660","cluster_addr":"172.21.15.156:6813/915951660","heartbeat_back_addr":"172.21.15.156:6817/915951660","heartbeat_front_addr":"172.21.15.156:6815/915951660","state":["exists","up"]},{"osd":2,"uuid":"d0051bbe-c0cd-42f4-9d8c-6b0a3e71886d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":88,"up_from":92,"up_thru":102,"down_at":89,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":3864636890},{"type":"v1","addr":"172.21.15.156:6819","nonce":3864636890}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":3864636890},{"type":"v1","addr":"172.21.15.156:6821","nonce":3864636890}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":3864636890},{"type":"v1","addr":"172.21.15.156:6825","nonce":3864636890}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":3864636890},{"type":"v1","addr":"172.21.15.156:6823","nonce":3864636890}]},"public_addr":"172.21.15.156:6819/3864636890","cluster_addr":"172.21.15.156:6821/3864636890","heartbeat_back_addr":"172.21.15.156:6825/3864636890","heartbeat_front_addr":"172.21.15.156:6823/3864636890","state":["exists","up"]},{"osd":3,"uuid":"479e109c-1e4a-4124-9442-f85d63df6d04","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":93,"up_from":97,"up_thru":102,"down_at":94,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":2037250231},{"type":"v1","addr":"172.21.15.156:6827","nonce":2037250231}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":2037250231},{"type":"v1","addr":"172.21.15.156:6829","nonce":2037250231}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6832","nonce":2037250231},{"type":"v1","addr":"172.21.15.156:6833","nonce":2037250231}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":2037250231},{"type":"v1","addr":"172.21.15.156:6831","nonce":2037250231}]},"public_addr":"172.21.15.156:6827/2037250231","cluster_addr":"172.21.15.156:6829/2037250231","heartbeat_back_addr":"172.21.15.156:6833/2037250231","heartbeat_front_addr":"172.21.15.156:6831/2037250231","state":["exists","up"]},{"osd":4,"uuid":"b090dbe4-ed23-4015-a864-074aa64cace4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":98,"up_from":102,"up_thru":102,"down_at":99,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":1502285480},{"type":"v1","addr":"172.21.15.171:6801","nonce":1502285480}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":1502285480},{"type":"v1","addr":"172.21.15.171:6803","nonce":1502285480}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":1502285480},{"type":"v1","addr":"172.21.15.171:6807","nonce":1502285480}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":1502285480},{"type":"v1","addr":"172.21.15.171:6805","nonce":1502285480}]},"public_addr":"172.21.15.171:6801/1502285480","cluster_addr":"172.21.15.171:6803/1502285480","heartbeat_back_addr":"172.21.15.171:6807/1502285480","heartbeat_front_addr":"172.21.15.171:6805/1502285480","state":["exists","up"]},{"osd":5,"uuid":"f506481f-428b-40d6-a54b-8ef411672b53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":102,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6809","nonce":2373798416}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6811","nonce":2373798416}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6815","nonce":2373798416}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":2373798416},{"type":"v1","addr":"172.21.15.171:6813","nonce":2373798416}]},"public_addr":"172.21.15.171:6809/2373798416","cluster_addr":"172.21.15.171:6811/2373798416","heartbeat_back_addr":"172.21.15.171:6815/2373798416","heartbeat_front_addr":"172.21.15.171:6813/2373798416","state":["exists","up"]},{"osd":6,"uuid":"f1fd019a-372d-4b4e-bede-7659038df10a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":102,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6817","nonce":2328644410}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6819","nonce":2328644410}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6823","nonce":2328644410}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2328644410},{"type":"v1","addr":"172.21.15.171:6821","nonce":2328644410}]},"public_addr":"172.21.15.171:6817/2328644410","cluster_addr":"172.21.15.171:6819/2328644410","heartbeat_back_addr":"172.21.15.171:6823/2328644410","heartbeat_front_addr":"172.21.15.171:6821/2328644410","state":["exists","up"]},{"osd":7,"uuid":"bee1398e-7912-458a-9f8b-279b84a22566","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":102,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6825","nonce":2507226151}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6827","nonce":2507226151}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6831","nonce":2507226151}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":2507226151},{"type":"v1","addr":"172.21.15.171:6829","nonce":2507226151}]},"public_addr":"172.21.15.171:6825/2507226151","cluster_addr":"172.21.15.171:6827/2507226151","heartbeat_back_addr":"172.21.15.171:6831/2507226151","heartbeat_front_addr":"172.21.15.171:6829/2507226151","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:03.400774+0000","dead_epoch":78},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:28.117542+0000","dead_epoch":83},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:52.563066+0000","dead_epoch":88},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:17.387470+0000","dead_epoch":93},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:40.460850+0000","dead_epoch":98},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:05.110287+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:29.602919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:55.143996+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.e","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:0/16376181":"2024-04-27T04:45:04.875889+0000","172.21.15.171:0/86196127":"2024-04-27T04:44:18.308381+0000","172.21.15.156:0/3331144037":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/402721227":"2024-04-27T04:34:42.908359+0000","172.21.15.171:0/4249015634":"2024-04-27T04:44:18.308381+0000","172.21.15.171:6835/2811112754":"2024-04-27T04:44:18.308381+0000","172.21.15.156:0/532521842":"2024-04-27T04:45:04.875889+0000","172.21.15.156:6801/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.171:6834/2811112754":"2024-04-27T04:44:18.308381+0000","172.21.15.171:0/4114228739":"2024-04-27T04:44:18.308381+0000","172.21.15.156:0/1323755296":"2024-04-27T04:45:04.875889+0000","172.21.15.156:0/3755168891":"2024-04-27T04:39:29.182492+0000","172.21.15.171:0/847470859":"2024-04-27T04:44:18.308381+0000","172.21.15.156:0/4047047180":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/1144243691":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/4143435073":"2024-04-27T04:34:18.740320+0000","172.21.15.156:0/711432262":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/67599134":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/2398977550":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6801/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:0/1361591211":"2024-04-27T04:43:10.958063+0000","172.21.15.156:0/1375023159":"2024-04-27T04:45:04.875889+0000","172.21.15.156:6800/367046062":"2024-04-27T04:43:10.958063+0000","172.21.15.156:6800/1096248653":"2024-04-27T04:34:42.908359+0000","172.21.15.156:6800/1084840701":"2024-04-27T04:39:29.182492+0000","172.21.15.156:6800/3726283619":"2024-04-27T04:45:04.875889+0000","172.21.15.156:6801/367046062":"2024-04-27T04:43:10.958063+0000","172.21.15.171:0/2399344582":"2024-04-27T04:44:18.308381+0000","172.21.15.156:0/314056408":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/32257782":"2024-04-27T04:43:10.958063+0000","172.21.15.156:0/3291954384":"2024-04-27T04:43:10.958063+0000","172.21.15.171:0/1847605205":"2024-04-27T04:44:18.308381+0000","172.21.15.156:6800/721789913":"2024-04-27T04:34:18.740320+0000","172.21.15.156:6801/1084840701":"2024-04-27T04:39:29.182492+0000","172.21.15.156:0/1772897570":"2024-04-27T04:43:10.958063+0000","172.21.15.156:0/2324075789":"2024-04-27T04:45:04.875889+0000","172.21.15.156:0/953980496":"2024-04-27T04:45:04.875889+0000","172.21.15.156:6801/3726283619":"2024-04-27T04:45:04.875889+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:52:01.303 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:01 smithi156 ceph-mon[98743]: pgmap v256: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:01.303 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:01 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/658859697' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:52:01.304 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:01 smithi156 ceph-mon[103290]: pgmap v256: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:01.304 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:01 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/658859697' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:52:01.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:01 smithi171 ceph-mon[90566]: pgmap v256: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:01.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:01 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/658859697' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:52:01.605 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-26T04:52:01.606 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-26T04:52:01.606 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-26T04:52:01.606 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-26T04:52:01.607 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-26T04:52:01.607 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-26T04:52:01.608 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-26T04:52:01.608 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-26T04:52:02.080 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:02.083 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:02.106 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:02.109 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:02.109 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:02.118 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:02.120 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:02.135 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:03.716 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:03 smithi156 ceph-mon[98743]: pgmap v257: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:03.717 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:03 smithi156 ceph-mon[103290]: pgmap v257: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:03.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:03 smithi171 ceph-mon[90566]: pgmap v257: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:04.830 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:52:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:52:04] "GET /metrics HTTP/1.1" 200 33900 "" "Prometheus/2.43.0" 2024-04-26T04:52:06.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:05 smithi171 ceph-mon[90566]: pgmap v258: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:06.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:05 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:52:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:05 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:06.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:05 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:52:06.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:05 smithi156 ceph-mon[98743]: pgmap v258: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:06.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:52:06.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:06.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:52:06.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:05 smithi156 ceph-mon[103290]: pgmap v258: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:06.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:05 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:52:06.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:05 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:06.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:05 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:52:08.232 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:07 smithi156 ceph-mon[98743]: pgmap v259: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:08.232 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:07 smithi156 ceph-mon[103290]: pgmap v259: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:08.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:07 smithi171 ceph-mon[90566]: pgmap v259: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:09.981 INFO:teuthology.orchestra.run.smithi156.stdout:352187318302 2024-04-26T04:52:09.982 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-26T04:52:10.214 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:09 smithi156 ceph-mon[98743]: pgmap v260: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:10.214 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:09 smithi156 ceph-mon[103290]: pgmap v260: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:10.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:09 smithi171 ceph-mon[90566]: pgmap v260: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:12.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:11 smithi171 ceph-mon[90566]: pgmap v261: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:12.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:11 smithi156 ceph-mon[98743]: pgmap v261: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:12.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:11 smithi156 ceph-mon[103290]: pgmap v261: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:12.464 INFO:teuthology.orchestra.run.smithi156.stdout:416611827721 2024-04-26T04:52:12.465 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-26T04:52:13.301 INFO:teuthology.orchestra.run.smithi156.stdout:163208757420 2024-04-26T04:52:13.301 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-26T04:52:14.200 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:13 smithi156 ceph-mon[98743]: pgmap v262: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:14.200 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:13 smithi156 ceph-mon[103290]: pgmap v262: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:14.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:13 smithi171 ceph-mon[90566]: pgmap v262: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:14.869 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:52:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:52:14] "GET /metrics HTTP/1.1" 200 33893 "" "Prometheus/2.43.0" 2024-04-26T04:52:16.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:15 smithi171 ceph-mon[90566]: pgmap v263: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:16.272 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:15 smithi156 ceph-mon[98743]: pgmap v263: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:16.273 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:15 smithi156 ceph-mon[103290]: pgmap v263: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:16.300 INFO:teuthology.orchestra.run.smithi156.stdout:231928234147 2024-04-26T04:52:16.301 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-26T04:52:17.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:17 smithi156 ceph-mon[103290]: pgmap v264: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:17.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:17 smithi156 ceph-mon[98743]: pgmap v264: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:17.430 INFO:teuthology.orchestra.run.smithi156.stdout:395136991249 2024-04-26T04:52:17.430 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-26T04:52:17.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:17 smithi171 ceph-mon[90566]: pgmap v264: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:17.889 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:17.889 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:17.893 INFO:teuthology.orchestra.run.smithi156.stdout:197568495784 2024-04-26T04:52:17.893 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-26T04:52:18.278 INFO:teuthology.orchestra.run.smithi156.stdout:438086664199 2024-04-26T04:52:18.278 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-26T04:52:19.727 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:19 smithi156 ceph-mon[103290]: pgmap v265: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:19.727 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:19 smithi156 ceph-mon[98743]: pgmap v265: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:19.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:19 smithi171 ceph-mon[90566]: pgmap v265: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:20.262 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:20.263 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:20.674 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:20.674 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:20.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:21.675 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:21 smithi156 ceph-mon[103290]: pgmap v266: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:21.675 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:21 smithi156 ceph-mon[98743]: pgmap v266: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:21.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:21 smithi171 ceph-mon[90566]: pgmap v266: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:22.651 INFO:teuthology.orchestra.run.smithi156.stdout:352187318305 2024-04-26T04:52:22.961 INFO:teuthology.orchestra.run.smithi156.stdout:373662154777 2024-04-26T04:52:22.962 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-26T04:52:23.398 INFO:teuthology.orchestra.run.smithi156.stdout:416611827724 2024-04-26T04:52:23.483 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:23 smithi156 ceph-mon[98743]: pgmap v267: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:23.483 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:23 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2751692542' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:52:23.483 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:23 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/336841955' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:52:23.483 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:23 smithi156 ceph-mon[103290]: pgmap v267: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:23.484 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:23 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2751692542' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:52:23.484 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:23 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/336841955' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:52:23.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:23 smithi171 ceph-mon[90566]: pgmap v267: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:23.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:23 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2751692542' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:52:23.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:23 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/336841955' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:52:24.103 INFO:tasks.cephadm.ceph_manager.ceph:need seq 352187318302 got 352187318305 for osd.0 2024-04-26T04:52:24.103 DEBUG:teuthology.parallel:result is None 2024-04-26T04:52:24.136 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:24.137 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:24.376 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:25.036 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:52:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:52:24] "GET /metrics HTTP/1.1" 200 33899 "" "Prometheus/2.43.0" 2024-04-26T04:52:25.345 INFO:tasks.cephadm.ceph_manager.ceph:need seq 416611827721 got 416611827724 for osd.3 2024-04-26T04:52:25.345 DEBUG:teuthology.parallel:result is None 2024-04-26T04:52:26.147 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:25 smithi156 ceph-mon[98743]: pgmap v268: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:26.147 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:25 smithi156 ceph-mon[103290]: pgmap v268: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:26.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:25 smithi171 ceph-mon[90566]: pgmap v268: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:27.832 INFO:teuthology.orchestra.run.smithi156.stdout:163208757424 2024-04-26T04:52:28.090 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:27 smithi156 ceph-mon[98743]: pgmap v269: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:28.090 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:27 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1199384783' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:52:28.091 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:27 smithi156 ceph-mon[103290]: pgmap v269: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:28.091 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:27 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1199384783' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:52:28.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:27 smithi171 ceph-mon[90566]: pgmap v269: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:28.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:27 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1199384783' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:52:29.652 INFO:teuthology.orchestra.run.smithi156.stdout:231928234150 2024-04-26T04:52:30.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:29 smithi156 ceph-mon[98743]: pgmap v270: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:30.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:29 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3307393091' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:52:30.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:29 smithi156 ceph-mon[103290]: pgmap v270: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:30.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:29 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3307393091' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:52:30.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:29 smithi171 ceph-mon[90566]: pgmap v270: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:30.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:29 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3307393091' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:52:30.702 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757420 got 163208757424 for osd.5 2024-04-26T04:52:30.702 DEBUG:teuthology.parallel:result is None 2024-04-26T04:52:31.120 INFO:teuthology.orchestra.run.smithi156.stdout:197568495787 2024-04-26T04:52:31.233 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:32.175 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:31 smithi156 ceph-mon[98743]: pgmap v271: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:32.175 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:31 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2063055851' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:52:32.176 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:31 smithi156 ceph-mon[103290]: pgmap v271: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:32.176 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:31 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2063055851' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:52:32.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:31 smithi171 ceph-mon[90566]: pgmap v271: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:32.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:31 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2063055851' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:52:32.553 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234147 got 231928234150 for osd.7 2024-04-26T04:52:32.553 DEBUG:teuthology.parallel:result is None 2024-04-26T04:52:33.167 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495784 got 197568495787 for osd.6 2024-04-26T04:52:33.167 DEBUG:teuthology.parallel:result is None 2024-04-26T04:52:33.851 INFO:teuthology.orchestra.run.smithi156.stdout:438086664202 2024-04-26T04:52:34.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:33 smithi156 ceph-mon[98743]: pgmap v272: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:34.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:33 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/998270918' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:52:34.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:33 smithi156 ceph-mon[103290]: pgmap v272: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:34.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:33 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/998270918' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:52:34.156 INFO:teuthology.orchestra.run.smithi156.stdout:395136991253 2024-04-26T04:52:34.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:33 smithi171 ceph-mon[90566]: pgmap v272: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:34.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:33 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/998270918' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:52:35.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:34 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1930884199' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:52:35.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:52:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:52:34] "GET /metrics HTTP/1.1" 200 33899 "" "Prometheus/2.43.0" 2024-04-26T04:52:35.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:34 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1930884199' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:52:35.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:34 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1930884199' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:52:35.656 INFO:tasks.cephadm.ceph_manager.ceph:need seq 438086664199 got 438086664202 for osd.4 2024-04-26T04:52:35.656 DEBUG:teuthology.parallel:result is None 2024-04-26T04:52:36.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:35 smithi171 ceph-mon[90566]: pgmap v273: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:36.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:35 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:36.269 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:35 smithi156 ceph-mon[98743]: pgmap v273: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:36.269 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:36.269 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:35 smithi156 ceph-mon[103290]: pgmap v273: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:36.269 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:36.338 INFO:teuthology.orchestra.run.smithi156.stdout:373662154780 2024-04-26T04:52:36.602 INFO:tasks.cephadm.ceph_manager.ceph:need seq 395136991249 got 395136991253 for osd.2 2024-04-26T04:52:36.602 DEBUG:teuthology.parallel:result is None 2024-04-26T04:52:37.207 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:36 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/586935278' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:52:37.208 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:36 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/586935278' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:52:37.234 INFO:tasks.cephadm.ceph_manager.ceph:need seq 373662154777 got 373662154780 for osd.1 2024-04-26T04:52:37.234 DEBUG:teuthology.parallel:result is None 2024-04-26T04:52:37.234 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-26T04:52:37.235 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:52:37.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:36 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/586935278' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:52:37.491 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:38.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:37 smithi171 ceph-mon[90566]: pgmap v274: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:38.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:37 smithi156 ceph-mon[103290]: pgmap v274: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:38.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:37 smithi156 ceph-mon[98743]: pgmap v274: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:39.407 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:52:39.408 INFO:teuthology.orchestra.run.smithi156.stderr:dumped all 2024-04-26T04:52:40.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:39 smithi156 ceph-mon[98743]: pgmap v275: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:40.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:39 smithi156 ceph-mon[103290]: pgmap v275: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:40.155 INFO:teuthology.orchestra.run.smithi156.stdout:{"pg_ready":true,"pg_map":{"version":275,"stamp":"2024-04-26T04:52:38.348329+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1077448,"kb_used_data":10628,"kb_used_omap":109,"kb_used_meta":1066578,"kb_avail":748883768,"statfs":{"total":767960285184,"available":766856978432,"internally_reserved":0,"allocated":10883072,"data_stored":6498366,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":111791,"internal_metadata":1092176721},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003015"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":61,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.978555+0000","last_change":"2024-04-26T04:51:53.978555+0000","last_active":"2024-04-26T04:51:53.978555+0000","last_peered":"2024-04-26T04:51:53.978555+0000","last_clean":"2024-04-26T04:51:53.978555+0000","last_became_active":"2024-04-26T04:51:53.977847+0000","last_became_peered":"2024-04-26T04:51:53.977847+0000","last_unstale":"2024-04-26T04:51:53.978555+0000","last_undegraded":"2024-04-26T04:51:53.978555+0000","last_fullsized":"2024-04-26T04:51:53.978555+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:41:01.980360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":74,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967122+0000","last_change":"2024-04-26T04:50:57.888980+0000","last_active":"2024-04-26T04:51:53.967122+0000","last_peered":"2024-04-26T04:51:53.967122+0000","last_clean":"2024-04-26T04:51:53.967122+0000","last_became_active":"2024-04-26T04:50:57.888704+0000","last_became_peered":"2024-04-26T04:50:57.888704+0000","last_unstale":"2024-04-26T04:51:53.967122+0000","last_undegraded":"2024-04-26T04:51:53.967122+0000","last_fullsized":"2024-04-26T04:51:53.967122+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:44:48.994332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.970464+0000","last_change":"2024-04-26T04:51:32.826710+0000","last_active":"2024-04-26T04:51:53.970464+0000","last_peered":"2024-04-26T04:51:53.970464+0000","last_clean":"2024-04-26T04:51:53.970464+0000","last_became_active":"2024-04-26T04:51:32.822552+0000","last_became_peered":"2024-04-26T04:51:32.822552+0000","last_unstale":"2024-04-26T04:51:53.970464+0000","last_undegraded":"2024-04-26T04:51:53.970464+0000","last_fullsized":"2024-04-26T04:51:53.970464+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:38:35.490921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":77,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.971482+0000","last_change":"2024-04-26T04:51:32.826437+0000","last_active":"2024-04-26T04:51:53.971482+0000","last_peered":"2024-04-26T04:51:53.971482+0000","last_clean":"2024-04-26T04:51:53.971482+0000","last_became_active":"2024-04-26T04:51:32.826089+0000","last_became_peered":"2024-04-26T04:51:32.826089+0000","last_unstale":"2024-04-26T04:51:53.971482+0000","last_undegraded":"2024-04-26T04:51:53.971482+0000","last_fullsized":"2024-04-26T04:51:53.971482+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:29:20.683183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":87,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.974922+0000","last_change":"2024-04-26T04:51:53.974922+0000","last_active":"2024-04-26T04:51:53.974922+0000","last_peered":"2024-04-26T04:51:53.974922+0000","last_clean":"2024-04-26T04:51:53.974922+0000","last_became_active":"2024-04-26T04:51:53.974600+0000","last_became_peered":"2024-04-26T04:51:53.974600+0000","last_unstale":"2024-04-26T04:51:53.974922+0000","last_undegraded":"2024-04-26T04:51:53.974922+0000","last_fullsized":"2024-04-26T04:51:53.974922+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:32:49.513479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":65,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.976347+0000","last_change":"2024-04-26T04:51:53.976347+0000","last_active":"2024-04-26T04:51:53.976347+0000","last_peered":"2024-04-26T04:51:53.976347+0000","last_clean":"2024-04-26T04:51:53.976347+0000","last_became_active":"2024-04-26T04:51:53.976002+0000","last_became_peered":"2024-04-26T04:51:53.976002+0000","last_unstale":"2024-04-26T04:51:53.976347+0000","last_undegraded":"2024-04-26T04:51:53.976347+0000","last_fullsized":"2024-04-26T04:51:53.976347+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:02:30.091927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":87,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.979147+0000","last_change":"2024-04-26T04:51:53.979147+0000","last_active":"2024-04-26T04:51:53.979147+0000","last_peered":"2024-04-26T04:51:53.979147+0000","last_clean":"2024-04-26T04:51:53.979147+0000","last_became_active":"2024-04-26T04:51:53.978465+0000","last_became_peered":"2024-04-26T04:51:53.978465+0000","last_unstale":"2024-04-26T04:51:53.979147+0000","last_undegraded":"2024-04-26T04:51:53.979147+0000","last_fullsized":"2024-04-26T04:51:53.979147+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:06:00.570187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":82,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.975391+0000","last_change":"2024-04-26T04:51:53.975391+0000","last_active":"2024-04-26T04:51:53.975391+0000","last_peered":"2024-04-26T04:51:53.975391+0000","last_clean":"2024-04-26T04:51:53.975391+0000","last_became_active":"2024-04-26T04:51:53.975041+0000","last_became_peered":"2024-04-26T04:51:53.975041+0000","last_unstale":"2024-04-26T04:51:53.975391+0000","last_undegraded":"2024-04-26T04:51:53.975391+0000","last_fullsized":"2024-04-26T04:51:53.975391+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:05:37.569828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":88,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.964401+0000","last_change":"2024-04-26T04:51:32.822901+0000","last_active":"2024-04-26T04:51:53.964401+0000","last_peered":"2024-04-26T04:51:53.964401+0000","last_clean":"2024-04-26T04:51:53.964401+0000","last_became_active":"2024-04-26T04:51:32.822555+0000","last_became_peered":"2024-04-26T04:51:32.822555+0000","last_unstale":"2024-04-26T04:51:53.964401+0000","last_undegraded":"2024-04-26T04:51:53.964401+0000","last_fullsized":"2024-04-26T04:51:53.964401+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:06:22.817233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":74,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.976203+0000","last_change":"2024-04-26T04:51:53.975318+0000","last_active":"2024-04-26T04:51:53.976203+0000","last_peered":"2024-04-26T04:51:53.976203+0000","last_clean":"2024-04-26T04:51:53.976203+0000","last_became_active":"2024-04-26T04:51:53.974785+0000","last_became_peered":"2024-04-26T04:51:53.974785+0000","last_unstale":"2024-04-26T04:51:53.976203+0000","last_undegraded":"2024-04-26T04:51:53.976203+0000","last_fullsized":"2024-04-26T04:51:53.976203+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:55:04.825082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":22,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.980442+0000","last_change":"2024-04-26T04:51:53.980442+0000","last_active":"2024-04-26T04:51:53.980442+0000","last_peered":"2024-04-26T04:51:53.980442+0000","last_clean":"2024-04-26T04:51:53.980442+0000","last_became_active":"2024-04-26T04:51:53.979841+0000","last_became_peered":"2024-04-26T04:51:53.979841+0000","last_unstale":"2024-04-26T04:51:53.980442+0000","last_undegraded":"2024-04-26T04:51:53.980442+0000","last_fullsized":"2024-04-26T04:51:53.980442+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:23:38.384902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":83,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.976225+0000","last_change":"2024-04-26T04:51:53.975866+0000","last_active":"2024-04-26T04:51:53.976225+0000","last_peered":"2024-04-26T04:51:53.976225+0000","last_clean":"2024-04-26T04:51:53.976225+0000","last_became_active":"2024-04-26T04:51:53.975460+0000","last_became_peered":"2024-04-26T04:51:53.975460+0000","last_unstale":"2024-04-26T04:51:53.976225+0000","last_undegraded":"2024-04-26T04:51:53.976225+0000","last_fullsized":"2024-04-26T04:51:53.976225+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:31:34.336852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.969260+0000","last_change":"2024-04-26T04:50:57.888468+0000","last_active":"2024-04-26T04:51:52.969260+0000","last_peered":"2024-04-26T04:51:52.969260+0000","last_clean":"2024-04-26T04:51:52.969260+0000","last_became_active":"2024-04-26T04:50:57.888090+0000","last_became_peered":"2024-04-26T04:50:57.888090+0000","last_unstale":"2024-04-26T04:51:52.969260+0000","last_undegraded":"2024-04-26T04:51:52.969260+0000","last_fullsized":"2024-04-26T04:51:52.969260+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:13:00.552525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":66,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.970255+0000","last_change":"2024-04-26T04:51:32.825735+0000","last_active":"2024-04-26T04:51:53.970255+0000","last_peered":"2024-04-26T04:51:53.970255+0000","last_clean":"2024-04-26T04:51:53.970255+0000","last_became_active":"2024-04-26T04:51:32.825418+0000","last_became_peered":"2024-04-26T04:51:32.825418+0000","last_unstale":"2024-04-26T04:51:53.970255+0000","last_undegraded":"2024-04-26T04:51:53.970255+0000","last_fullsized":"2024-04-26T04:51:53.970255+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:58:10.738522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":61,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.975994+0000","last_change":"2024-04-26T04:51:53.975487+0000","last_active":"2024-04-26T04:51:53.975994+0000","last_peered":"2024-04-26T04:51:53.975994+0000","last_clean":"2024-04-26T04:51:53.975994+0000","last_became_active":"2024-04-26T04:51:53.975187+0000","last_became_peered":"2024-04-26T04:51:53.975187+0000","last_unstale":"2024-04-26T04:51:53.975994+0000","last_undegraded":"2024-04-26T04:51:53.975994+0000","last_fullsized":"2024-04-26T04:51:53.975994+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:49:35.166974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.971481+0000","last_change":"2024-04-26T04:51:32.828752+0000","last_active":"2024-04-26T04:51:53.971481+0000","last_peered":"2024-04-26T04:51:53.971481+0000","last_clean":"2024-04-26T04:51:53.971481+0000","last_became_active":"2024-04-26T04:51:32.826649+0000","last_became_peered":"2024-04-26T04:51:32.826649+0000","last_unstale":"2024-04-26T04:51:53.971481+0000","last_undegraded":"2024-04-26T04:51:53.971481+0000","last_fullsized":"2024-04-26T04:51:53.971481+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:37:03.290982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":62,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.966918+0000","last_change":"2024-04-26T04:49:48.625191+0000","last_active":"2024-04-26T04:51:53.966918+0000","last_peered":"2024-04-26T04:51:53.966918+0000","last_clean":"2024-04-26T04:51:53.966918+0000","last_became_active":"2024-04-26T04:49:48.624193+0000","last_became_peered":"2024-04-26T04:49:48.624193+0000","last_unstale":"2024-04-26T04:51:53.966918+0000","last_undegraded":"2024-04-26T04:51:53.966918+0000","last_fullsized":"2024-04-26T04:51:53.966918+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:01:56.529663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":61,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967526+0000","last_change":"2024-04-26T04:50:57.888580+0000","last_active":"2024-04-26T04:51:53.967526+0000","last_peered":"2024-04-26T04:51:53.967526+0000","last_clean":"2024-04-26T04:51:53.967526+0000","last_became_active":"2024-04-26T04:50:57.888273+0000","last_became_peered":"2024-04-26T04:50:57.888273+0000","last_unstale":"2024-04-26T04:51:53.967526+0000","last_undegraded":"2024-04-26T04:51:53.967526+0000","last_fullsized":"2024-04-26T04:51:53.967526+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:01:15.472987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":84,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.974410+0000","last_change":"2024-04-26T04:51:53.974410+0000","last_active":"2024-04-26T04:51:53.974410+0000","last_peered":"2024-04-26T04:51:53.974410+0000","last_clean":"2024-04-26T04:51:53.974410+0000","last_became_active":"2024-04-26T04:51:53.973906+0000","last_became_peered":"2024-04-26T04:51:53.973906+0000","last_unstale":"2024-04-26T04:51:53.974410+0000","last_undegraded":"2024-04-26T04:51:53.974410+0000","last_fullsized":"2024-04-26T04:51:53.974410+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:18:42.256975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":80,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967276+0000","last_change":"2024-04-26T04:50:57.888162+0000","last_active":"2024-04-26T04:51:53.967276+0000","last_peered":"2024-04-26T04:51:53.967276+0000","last_clean":"2024-04-26T04:51:53.967276+0000","last_became_active":"2024-04-26T04:50:57.887900+0000","last_became_peered":"2024-04-26T04:50:57.887900+0000","last_unstale":"2024-04-26T04:51:53.967276+0000","last_undegraded":"2024-04-26T04:51:53.967276+0000","last_fullsized":"2024-04-26T04:51:53.967276+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:59:30.563174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":80,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.975006+0000","last_change":"2024-04-26T04:51:53.975006+0000","last_active":"2024-04-26T04:51:53.975006+0000","last_peered":"2024-04-26T04:51:53.975006+0000","last_clean":"2024-04-26T04:51:53.975006+0000","last_became_active":"2024-04-26T04:51:53.974717+0000","last_became_peered":"2024-04-26T04:51:53.974717+0000","last_unstale":"2024-04-26T04:51:53.975006+0000","last_undegraded":"2024-04-26T04:51:53.975006+0000","last_fullsized":"2024-04-26T04:51:53.975006+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:57:27.538723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":84,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.970133+0000","last_change":"2024-04-26T04:50:23.338298+0000","last_active":"2024-04-26T04:51:52.970133+0000","last_peered":"2024-04-26T04:51:52.970133+0000","last_clean":"2024-04-26T04:51:52.970133+0000","last_became_active":"2024-04-26T04:50:23.337891+0000","last_became_peered":"2024-04-26T04:50:23.337891+0000","last_unstale":"2024-04-26T04:51:52.970133+0000","last_undegraded":"2024-04-26T04:51:52.970133+0000","last_fullsized":"2024-04-26T04:51:52.970133+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:01:34.076451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":63,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.977282+0000","last_change":"2024-04-26T04:51:53.977282+0000","last_active":"2024-04-26T04:51:53.977282+0000","last_peered":"2024-04-26T04:51:53.977282+0000","last_clean":"2024-04-26T04:51:53.977282+0000","last_became_active":"2024-04-26T04:51:53.976853+0000","last_became_peered":"2024-04-26T04:51:53.976853+0000","last_unstale":"2024-04-26T04:51:53.977282+0000","last_undegraded":"2024-04-26T04:51:53.977282+0000","last_fullsized":"2024-04-26T04:51:53.977282+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:26:56.909521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":32,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.970310+0000","last_change":"2024-04-26T04:51:32.829167+0000","last_active":"2024-04-26T04:51:53.970310+0000","last_peered":"2024-04-26T04:51:53.970310+0000","last_clean":"2024-04-26T04:51:53.970310+0000","last_became_active":"2024-04-26T04:51:32.827225+0000","last_became_peered":"2024-04-26T04:51:32.827225+0000","last_unstale":"2024-04-26T04:51:53.970310+0000","last_undegraded":"2024-04-26T04:51:53.970310+0000","last_fullsized":"2024-04-26T04:51:53.970310+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:00:26.343050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":442,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.970048+0000","last_change":"2024-04-26T04:49:48.625214+0000","last_active":"2024-04-26T04:51:52.970048+0000","last_peered":"2024-04-26T04:51:52.970048+0000","last_clean":"2024-04-26T04:51:52.970048+0000","last_became_active":"2024-04-26T04:49:48.624784+0000","last_became_peered":"2024-04-26T04:49:48.624784+0000","last_unstale":"2024-04-26T04:51:52.970048+0000","last_undegraded":"2024-04-26T04:51:52.970048+0000","last_fullsized":"2024-04-26T04:51:52.970048+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:36:55.213599+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:36:55.213599+0000","last_clean_scrub_stamp":"2024-04-26T04:36:55.213599+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:33:02.237214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":68,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.964303+0000","last_change":"2024-04-26T04:50:57.889224+0000","last_active":"2024-04-26T04:51:52.964303+0000","last_peered":"2024-04-26T04:51:52.964303+0000","last_clean":"2024-04-26T04:51:52.964303+0000","last_became_active":"2024-04-26T04:50:57.888860+0000","last_became_peered":"2024-04-26T04:50:57.888860+0000","last_unstale":"2024-04-26T04:51:52.964303+0000","last_undegraded":"2024-04-26T04:51:52.964303+0000","last_fullsized":"2024-04-26T04:51:52.964303+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:25:02.199086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":85,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967916+0000","last_change":"2024-04-26T04:50:23.340171+0000","last_active":"2024-04-26T04:51:53.967916+0000","last_peered":"2024-04-26T04:51:53.967916+0000","last_clean":"2024-04-26T04:51:53.967916+0000","last_became_active":"2024-04-26T04:50:23.338209+0000","last_became_peered":"2024-04-26T04:50:23.338209+0000","last_unstale":"2024-04-26T04:51:53.967916+0000","last_undegraded":"2024-04-26T04:51:53.967916+0000","last_fullsized":"2024-04-26T04:51:53.967916+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:54:07.999658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":61,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967239+0000","last_change":"2024-04-26T04:50:57.888370+0000","last_active":"2024-04-26T04:51:53.967239+0000","last_peered":"2024-04-26T04:51:53.967239+0000","last_clean":"2024-04-26T04:51:53.967239+0000","last_became_active":"2024-04-26T04:50:57.888063+0000","last_became_peered":"2024-04-26T04:50:57.888063+0000","last_unstale":"2024-04-26T04:51:53.967239+0000","last_undegraded":"2024-04-26T04:51:53.967239+0000","last_fullsized":"2024-04-26T04:51:53.967239+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:41:15.343980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":90,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.964325+0000","last_change":"2024-04-26T04:51:32.830115+0000","last_active":"2024-04-26T04:51:53.964325+0000","last_peered":"2024-04-26T04:51:53.964325+0000","last_clean":"2024-04-26T04:51:53.964325+0000","last_became_active":"2024-04-26T04:51:32.829777+0000","last_became_peered":"2024-04-26T04:51:32.829777+0000","last_unstale":"2024-04-26T04:51:53.964325+0000","last_undegraded":"2024-04-26T04:51:53.964325+0000","last_fullsized":"2024-04-26T04:51:53.964325+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:48:16.578287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967621+0000","last_change":"2024-04-26T04:51:32.825033+0000","last_active":"2024-04-26T04:51:53.967621+0000","last_peered":"2024-04-26T04:51:53.967621+0000","last_clean":"2024-04-26T04:51:53.967621+0000","last_became_active":"2024-04-26T04:51:32.824585+0000","last_became_peered":"2024-04-26T04:51:32.824585+0000","last_unstale":"2024-04-26T04:51:53.967621+0000","last_undegraded":"2024-04-26T04:51:53.967621+0000","last_fullsized":"2024-04-26T04:51:53.967621+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:04:29.404053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":57,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.980224+0000","last_change":"2024-04-26T04:51:53.980224+0000","last_active":"2024-04-26T04:51:53.980224+0000","last_peered":"2024-04-26T04:51:53.980224+0000","last_clean":"2024-04-26T04:51:53.980224+0000","last_became_active":"2024-04-26T04:51:53.979629+0000","last_became_peered":"2024-04-26T04:51:53.979629+0000","last_unstale":"2024-04-26T04:51:53.980224+0000","last_undegraded":"2024-04-26T04:51:53.980224+0000","last_fullsized":"2024-04-26T04:51:53.980224+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:02:25.756702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":68,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.965367+0000","last_change":"2024-04-26T04:50:23.336008+0000","last_active":"2024-04-26T04:51:52.965367+0000","last_peered":"2024-04-26T04:51:52.965367+0000","last_clean":"2024-04-26T04:51:52.965367+0000","last_became_active":"2024-04-26T04:50:23.335654+0000","last_became_peered":"2024-04-26T04:50:23.335654+0000","last_unstale":"2024-04-26T04:51:52.965367+0000","last_undegraded":"2024-04-26T04:51:52.965367+0000","last_fullsized":"2024-04-26T04:51:52.965367+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:50:03.658838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":52,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.970795+0000","last_change":"2024-04-26T04:51:53.970795+0000","last_active":"2024-04-26T04:51:53.970795+0000","last_peered":"2024-04-26T04:51:53.970795+0000","last_clean":"2024-04-26T04:51:53.970795+0000","last_became_active":"2024-04-26T04:51:53.970480+0000","last_became_peered":"2024-04-26T04:51:53.970480+0000","last_unstale":"2024-04-26T04:51:53.970795+0000","last_undegraded":"2024-04-26T04:51:53.970795+0000","last_fullsized":"2024-04-26T04:51:53.970795+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:16:36.478106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":80,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.964687+0000","last_change":"2024-04-26T04:51:32.827447+0000","last_active":"2024-04-26T04:51:52.964687+0000","last_peered":"2024-04-26T04:51:52.964687+0000","last_clean":"2024-04-26T04:51:52.964687+0000","last_became_active":"2024-04-26T04:51:32.826909+0000","last_became_peered":"2024-04-26T04:51:32.826909+0000","last_unstale":"2024-04-26T04:51:52.964687+0000","last_undegraded":"2024-04-26T04:51:52.964687+0000","last_fullsized":"2024-04-26T04:51:52.964687+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:08:54.742719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":91,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967429+0000","last_change":"2024-04-26T04:50:23.340374+0000","last_active":"2024-04-26T04:51:53.967429+0000","last_peered":"2024-04-26T04:51:53.967429+0000","last_clean":"2024-04-26T04:51:53.967429+0000","last_became_active":"2024-04-26T04:50:23.337021+0000","last_became_peered":"2024-04-26T04:50:23.337021+0000","last_unstale":"2024-04-26T04:51:53.967429+0000","last_undegraded":"2024-04-26T04:51:53.967429+0000","last_fullsized":"2024-04-26T04:51:53.967429+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:11:12.157249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":61,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.975022+0000","last_change":"2024-04-26T04:51:53.975022+0000","last_active":"2024-04-26T04:51:53.975022+0000","last_peered":"2024-04-26T04:51:53.975022+0000","last_clean":"2024-04-26T04:51:53.975022+0000","last_became_active":"2024-04-26T04:51:53.974709+0000","last_became_peered":"2024-04-26T04:51:53.974709+0000","last_unstale":"2024-04-26T04:51:53.975022+0000","last_undegraded":"2024-04-26T04:51:53.975022+0000","last_fullsized":"2024-04-26T04:51:53.975022+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:15:46.651017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.970349+0000","last_change":"2024-04-26T04:51:32.827981+0000","last_active":"2024-04-26T04:51:53.970349+0000","last_peered":"2024-04-26T04:51:53.970349+0000","last_clean":"2024-04-26T04:51:53.970349+0000","last_became_active":"2024-04-26T04:51:32.827654+0000","last_became_peered":"2024-04-26T04:51:32.827654+0000","last_unstale":"2024-04-26T04:51:53.970349+0000","last_undegraded":"2024-04-26T04:51:53.970349+0000","last_fullsized":"2024-04-26T04:51:53.970349+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:24:04.308350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":35,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.980388+0000","last_change":"2024-04-26T04:51:53.980388+0000","last_active":"2024-04-26T04:51:53.980388+0000","last_peered":"2024-04-26T04:51:53.980388+0000","last_clean":"2024-04-26T04:51:53.980388+0000","last_became_active":"2024-04-26T04:51:53.979784+0000","last_became_peered":"2024-04-26T04:51:53.979784+0000","last_unstale":"2024-04-26T04:51:53.980388+0000","last_undegraded":"2024-04-26T04:51:53.980388+0000","last_fullsized":"2024-04-26T04:51:53.980388+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:53:46.073564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":67,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.976228+0000","last_change":"2024-04-26T04:51:53.976228+0000","last_active":"2024-04-26T04:51:53.976228+0000","last_peered":"2024-04-26T04:51:53.976228+0000","last_clean":"2024-04-26T04:51:53.976228+0000","last_became_active":"2024-04-26T04:51:53.975872+0000","last_became_peered":"2024-04-26T04:51:53.975872+0000","last_unstale":"2024-04-26T04:51:53.976228+0000","last_undegraded":"2024-04-26T04:51:53.976228+0000","last_fullsized":"2024-04-26T04:51:53.976228+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:04:19.797962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":83,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.977758+0000","last_change":"2024-04-26T04:51:53.977758+0000","last_active":"2024-04-26T04:51:53.977758+0000","last_peered":"2024-04-26T04:51:53.977758+0000","last_clean":"2024-04-26T04:51:53.977758+0000","last_became_active":"2024-04-26T04:51:53.977117+0000","last_became_peered":"2024-04-26T04:51:53.977117+0000","last_unstale":"2024-04-26T04:51:53.977758+0000","last_undegraded":"2024-04-26T04:51:53.977758+0000","last_fullsized":"2024-04-26T04:51:53.977758+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:46:42.115079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":81,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.977884+0000","last_change":"2024-04-26T04:51:53.977884+0000","last_active":"2024-04-26T04:51:53.977884+0000","last_peered":"2024-04-26T04:51:53.977884+0000","last_clean":"2024-04-26T04:51:53.977884+0000","last_became_active":"2024-04-26T04:51:53.977583+0000","last_became_peered":"2024-04-26T04:51:53.977583+0000","last_unstale":"2024-04-26T04:51:53.977884+0000","last_undegraded":"2024-04-26T04:51:53.977884+0000","last_fullsized":"2024-04-26T04:51:53.977884+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:14:49.749925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.980287+0000","last_change":"2024-04-26T04:51:53.980287+0000","last_active":"2024-04-26T04:51:53.980287+0000","last_peered":"2024-04-26T04:51:53.980287+0000","last_clean":"2024-04-26T04:51:53.980287+0000","last_became_active":"2024-04-26T04:51:53.979720+0000","last_became_peered":"2024-04-26T04:51:53.979720+0000","last_unstale":"2024-04-26T04:51:53.980287+0000","last_undegraded":"2024-04-26T04:51:53.980287+0000","last_fullsized":"2024-04-26T04:51:53.980287+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:17:53.138239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.974532+0000","last_change":"2024-04-26T04:51:53.974532+0000","last_active":"2024-04-26T04:51:53.974532+0000","last_peered":"2024-04-26T04:51:53.974532+0000","last_clean":"2024-04-26T04:51:53.974532+0000","last_became_active":"2024-04-26T04:51:53.973947+0000","last_became_peered":"2024-04-26T04:51:53.973947+0000","last_unstale":"2024-04-26T04:51:53.974532+0000","last_undegraded":"2024-04-26T04:51:53.974532+0000","last_fullsized":"2024-04-26T04:51:53.974532+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:32:58.731977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":80,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.976335+0000","last_change":"2024-04-26T04:51:53.976012+0000","last_active":"2024-04-26T04:51:53.976335+0000","last_peered":"2024-04-26T04:51:53.976335+0000","last_clean":"2024-04-26T04:51:53.976335+0000","last_became_active":"2024-04-26T04:51:53.975654+0000","last_became_peered":"2024-04-26T04:51:53.975654+0000","last_unstale":"2024-04-26T04:51:53.976335+0000","last_undegraded":"2024-04-26T04:51:53.976335+0000","last_fullsized":"2024-04-26T04:51:53.976335+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:41:11.469823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":63,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.964436+0000","last_change":"2024-04-26T04:51:32.827338+0000","last_active":"2024-04-26T04:51:52.964436+0000","last_peered":"2024-04-26T04:51:52.964436+0000","last_clean":"2024-04-26T04:51:52.964436+0000","last_became_active":"2024-04-26T04:51:32.826934+0000","last_became_peered":"2024-04-26T04:51:32.826934+0000","last_unstale":"2024-04-26T04:51:52.964436+0000","last_undegraded":"2024-04-26T04:51:52.964436+0000","last_fullsized":"2024-04-26T04:51:52.964436+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:39:15.876554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":63,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.968009+0000","last_change":"2024-04-26T04:50:23.340925+0000","last_active":"2024-04-26T04:51:52.968009+0000","last_peered":"2024-04-26T04:51:52.968009+0000","last_clean":"2024-04-26T04:51:52.968009+0000","last_became_active":"2024-04-26T04:50:23.340605+0000","last_became_peered":"2024-04-26T04:50:23.340605+0000","last_unstale":"2024-04-26T04:51:52.968009+0000","last_undegraded":"2024-04-26T04:51:52.968009+0000","last_fullsized":"2024-04-26T04:51:52.968009+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:12:05.499630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":46,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.972685+0000","last_change":"2024-04-26T04:40:44.265256+0000","last_active":"2024-04-26T04:51:52.972685+0000","last_peered":"2024-04-26T04:51:52.972685+0000","last_clean":"2024-04-26T04:51:52.972685+0000","last_became_active":"2024-04-26T04:40:44.264777+0000","last_became_peered":"2024-04-26T04:40:44.264777+0000","last_unstale":"2024-04-26T04:51:52.972685+0000","last_undegraded":"2024-04-26T04:51:52.972685+0000","last_fullsized":"2024-04-26T04:51:52.972685+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:05:04.374938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":87,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.978735+0000","last_change":"2024-04-26T04:51:53.978735+0000","last_active":"2024-04-26T04:51:53.978735+0000","last_peered":"2024-04-26T04:51:53.978735+0000","last_clean":"2024-04-26T04:51:53.978735+0000","last_became_active":"2024-04-26T04:51:53.977433+0000","last_became_peered":"2024-04-26T04:51:53.977433+0000","last_unstale":"2024-04-26T04:51:53.978735+0000","last_undegraded":"2024-04-26T04:51:53.978735+0000","last_fullsized":"2024-04-26T04:51:53.978735+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:54:03.890519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":81,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.973442+0000","last_change":"2024-04-26T04:51:53.973442+0000","last_active":"2024-04-26T04:51:53.973442+0000","last_peered":"2024-04-26T04:51:53.973442+0000","last_clean":"2024-04-26T04:51:53.973442+0000","last_became_active":"2024-04-26T04:51:53.973098+0000","last_became_peered":"2024-04-26T04:51:53.973098+0000","last_unstale":"2024-04-26T04:51:53.973442+0000","last_undegraded":"2024-04-26T04:51:53.973442+0000","last_fullsized":"2024-04-26T04:51:53.973442+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:34:05.204900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":45,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.964231+0000","last_change":"2024-04-26T04:40:44.864902+0000","last_active":"2024-04-26T04:51:52.964231+0000","last_peered":"2024-04-26T04:51:52.964231+0000","last_clean":"2024-04-26T04:51:52.964231+0000","last_became_active":"2024-04-26T04:40:44.864603+0000","last_became_peered":"2024-04-26T04:40:44.864603+0000","last_unstale":"2024-04-26T04:51:52.964231+0000","last_undegraded":"2024-04-26T04:51:52.964231+0000","last_fullsized":"2024-04-26T04:51:52.964231+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:50:44.169231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":80,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.974364+0000","last_change":"2024-04-26T04:51:53.974364+0000","last_active":"2024-04-26T04:51:53.974364+0000","last_peered":"2024-04-26T04:51:53.974364+0000","last_clean":"2024-04-26T04:51:53.974364+0000","last_became_active":"2024-04-26T04:51:53.974031+0000","last_became_peered":"2024-04-26T04:51:53.974031+0000","last_unstale":"2024-04-26T04:51:53.974364+0000","last_undegraded":"2024-04-26T04:51:53.974364+0000","last_fullsized":"2024-04-26T04:51:53.974364+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:31:21.804373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.971391+0000","last_change":"2024-04-26T04:51:32.831989+0000","last_active":"2024-04-26T04:51:53.971391+0000","last_peered":"2024-04-26T04:51:53.971391+0000","last_clean":"2024-04-26T04:51:53.971391+0000","last_became_active":"2024-04-26T04:51:32.830987+0000","last_became_peered":"2024-04-26T04:51:32.830987+0000","last_unstale":"2024-04-26T04:51:53.971391+0000","last_undegraded":"2024-04-26T04:51:53.971391+0000","last_fullsized":"2024-04-26T04:51:53.971391+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:07:35.261462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":72,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.967042+0000","last_change":"2024-04-26T04:49:48.623488+0000","last_active":"2024-04-26T04:51:52.967042+0000","last_peered":"2024-04-26T04:51:52.967042+0000","last_clean":"2024-04-26T04:51:52.967042+0000","last_became_active":"2024-04-26T04:49:48.623070+0000","last_became_peered":"2024-04-26T04:49:48.623070+0000","last_unstale":"2024-04-26T04:51:52.967042+0000","last_undegraded":"2024-04-26T04:51:52.967042+0000","last_fullsized":"2024-04-26T04:51:52.967042+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:39:35.560811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.966562+0000","last_change":"2024-04-26T04:51:32.827894+0000","last_active":"2024-04-26T04:51:53.966562+0000","last_peered":"2024-04-26T04:51:53.966562+0000","last_clean":"2024-04-26T04:51:53.966562+0000","last_became_active":"2024-04-26T04:51:32.827492+0000","last_became_peered":"2024-04-26T04:51:32.827492+0000","last_unstale":"2024-04-26T04:51:53.966562+0000","last_undegraded":"2024-04-26T04:51:53.966562+0000","last_fullsized":"2024-04-26T04:51:53.966562+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:07:48.796063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":62,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.971090+0000","last_change":"2024-04-26T04:51:53.971090+0000","last_active":"2024-04-26T04:51:53.971090+0000","last_peered":"2024-04-26T04:51:53.971090+0000","last_clean":"2024-04-26T04:51:53.971090+0000","last_became_active":"2024-04-26T04:51:53.970806+0000","last_became_peered":"2024-04-26T04:51:53.970806+0000","last_unstale":"2024-04-26T04:51:53.971090+0000","last_undegraded":"2024-04-26T04:51:53.971090+0000","last_fullsized":"2024-04-26T04:51:53.971090+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:59:20.358643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":62,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.964605+0000","last_change":"2024-04-26T04:51:32.824417+0000","last_active":"2024-04-26T04:51:52.964605+0000","last_peered":"2024-04-26T04:51:52.964605+0000","last_clean":"2024-04-26T04:51:52.964605+0000","last_became_active":"2024-04-26T04:51:32.824013+0000","last_became_peered":"2024-04-26T04:51:32.824013+0000","last_unstale":"2024-04-26T04:51:52.964605+0000","last_undegraded":"2024-04-26T04:51:52.964605+0000","last_fullsized":"2024-04-26T04:51:52.964605+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:32:44.950039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.966601+0000","last_change":"2024-04-26T04:51:32.824357+0000","last_active":"2024-04-26T04:51:53.966601+0000","last_peered":"2024-04-26T04:51:53.966601+0000","last_clean":"2024-04-26T04:51:53.966601+0000","last_became_active":"2024-04-26T04:51:32.824033+0000","last_became_peered":"2024-04-26T04:51:32.824033+0000","last_unstale":"2024-04-26T04:51:53.966601+0000","last_undegraded":"2024-04-26T04:51:53.966601+0000","last_fullsized":"2024-04-26T04:51:53.966601+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:20:02.689022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.971378+0000","last_change":"2024-04-26T04:51:32.829027+0000","last_active":"2024-04-26T04:51:53.971378+0000","last_peered":"2024-04-26T04:51:53.971378+0000","last_clean":"2024-04-26T04:51:53.971378+0000","last_became_active":"2024-04-26T04:51:32.828359+0000","last_became_peered":"2024-04-26T04:51:32.828359+0000","last_unstale":"2024-04-26T04:51:53.971378+0000","last_undegraded":"2024-04-26T04:51:53.971378+0000","last_fullsized":"2024-04-26T04:51:53.971378+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:32:15.010228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":81,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.971881+0000","last_change":"2024-04-26T04:51:53.971881+0000","last_active":"2024-04-26T04:51:53.971881+0000","last_peered":"2024-04-26T04:51:53.971881+0000","last_clean":"2024-04-26T04:51:53.971881+0000","last_became_active":"2024-04-26T04:51:53.971550+0000","last_became_peered":"2024-04-26T04:51:53.971550+0000","last_unstale":"2024-04-26T04:51:53.971881+0000","last_undegraded":"2024-04-26T04:51:53.971881+0000","last_fullsized":"2024-04-26T04:51:53.971881+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:03:09.916815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":61,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.980248+0000","last_change":"2024-04-26T04:51:53.980248+0000","last_active":"2024-04-26T04:51:53.980248+0000","last_peered":"2024-04-26T04:51:53.980248+0000","last_clean":"2024-04-26T04:51:53.980248+0000","last_became_active":"2024-04-26T04:51:53.979699+0000","last_became_peered":"2024-04-26T04:51:53.979699+0000","last_unstale":"2024-04-26T04:51:53.980248+0000","last_undegraded":"2024-04-26T04:51:53.980248+0000","last_fullsized":"2024-04-26T04:51:53.980248+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:12:50.870138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"78'22","reported_seq":57,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.975142+0000","last_change":"2024-04-26T04:51:53.975142+0000","last_active":"2024-04-26T04:51:53.975142+0000","last_peered":"2024-04-26T04:51:53.975142+0000","last_clean":"2024-04-26T04:51:53.975142+0000","last_became_active":"2024-04-26T04:51:53.974790+0000","last_became_peered":"2024-04-26T04:51:53.974790+0000","last_unstale":"2024-04-26T04:51:53.975142+0000","last_undegraded":"2024-04-26T04:51:53.975142+0000","last_fullsized":"2024-04-26T04:51:53.975142+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:21:51.027054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":60,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967032+0000","last_change":"2024-04-26T04:50:57.888725+0000","last_active":"2024-04-26T04:51:53.967032+0000","last_peered":"2024-04-26T04:51:53.967032+0000","last_clean":"2024-04-26T04:51:53.967032+0000","last_became_active":"2024-04-26T04:50:57.887912+0000","last_became_peered":"2024-04-26T04:50:57.887912+0000","last_unstale":"2024-04-26T04:51:53.967032+0000","last_undegraded":"2024-04-26T04:51:53.967032+0000","last_fullsized":"2024-04-26T04:51:53.967032+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:03:54.843721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":82,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.974244+0000","last_change":"2024-04-26T04:51:53.974244+0000","last_active":"2024-04-26T04:51:53.974244+0000","last_peered":"2024-04-26T04:51:53.974244+0000","last_clean":"2024-04-26T04:51:53.974244+0000","last_became_active":"2024-04-26T04:51:53.973884+0000","last_became_peered":"2024-04-26T04:51:53.973884+0000","last_unstale":"2024-04-26T04:51:53.974244+0000","last_undegraded":"2024-04-26T04:51:53.974244+0000","last_fullsized":"2024-04-26T04:51:53.974244+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:52:16.159672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":32,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.970376+0000","last_change":"2024-04-26T04:51:32.830946+0000","last_active":"2024-04-26T04:51:53.970376+0000","last_peered":"2024-04-26T04:51:53.970376+0000","last_clean":"2024-04-26T04:51:53.970376+0000","last_became_active":"2024-04-26T04:51:32.829261+0000","last_became_peered":"2024-04-26T04:51:32.829261+0000","last_unstale":"2024-04-26T04:51:53.970376+0000","last_undegraded":"2024-04-26T04:51:53.970376+0000","last_fullsized":"2024-04-26T04:51:53.970376+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:11:33.805937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":63,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.964349+0000","last_change":"2024-04-26T04:50:23.339092+0000","last_active":"2024-04-26T04:51:52.964349+0000","last_peered":"2024-04-26T04:51:52.964349+0000","last_clean":"2024-04-26T04:51:52.964349+0000","last_became_active":"2024-04-26T04:50:23.338761+0000","last_became_peered":"2024-04-26T04:50:23.338761+0000","last_unstale":"2024-04-26T04:51:52.964349+0000","last_undegraded":"2024-04-26T04:51:52.964349+0000","last_fullsized":"2024-04-26T04:51:52.964349+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:55:21.983639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.969495+0000","last_change":"2024-04-26T04:51:32.829000+0000","last_active":"2024-04-26T04:51:53.969495+0000","last_peered":"2024-04-26T04:51:53.969495+0000","last_clean":"2024-04-26T04:51:53.969495+0000","last_became_active":"2024-04-26T04:51:32.828549+0000","last_became_peered":"2024-04-26T04:51:32.828549+0000","last_unstale":"2024-04-26T04:51:53.969495+0000","last_undegraded":"2024-04-26T04:51:53.969495+0000","last_fullsized":"2024-04-26T04:51:53.969495+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:43:34.903419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967258+0000","last_change":"2024-04-26T04:51:32.827496+0000","last_active":"2024-04-26T04:51:53.967258+0000","last_peered":"2024-04-26T04:51:53.967258+0000","last_clean":"2024-04-26T04:51:53.967258+0000","last_became_active":"2024-04-26T04:51:32.827199+0000","last_became_peered":"2024-04-26T04:51:32.827199+0000","last_unstale":"2024-04-26T04:51:53.967258+0000","last_undegraded":"2024-04-26T04:51:53.967258+0000","last_fullsized":"2024-04-26T04:51:53.967258+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:00:20.621104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":63,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.969337+0000","last_change":"2024-04-26T04:50:23.336719+0000","last_active":"2024-04-26T04:51:52.969337+0000","last_peered":"2024-04-26T04:51:52.969337+0000","last_clean":"2024-04-26T04:51:52.969337+0000","last_became_active":"2024-04-26T04:50:23.336382+0000","last_became_peered":"2024-04-26T04:50:23.336382+0000","last_unstale":"2024-04-26T04:51:52.969337+0000","last_undegraded":"2024-04-26T04:51:52.969337+0000","last_fullsized":"2024-04-26T04:51:52.969337+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:36:29.868029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.969660+0000","last_change":"2024-04-26T04:51:32.825815+0000","last_active":"2024-04-26T04:51:53.969660+0000","last_peered":"2024-04-26T04:51:53.969660+0000","last_clean":"2024-04-26T04:51:53.969660+0000","last_became_active":"2024-04-26T04:51:32.825485+0000","last_became_peered":"2024-04-26T04:51:32.825485+0000","last_unstale":"2024-04-26T04:51:53.969660+0000","last_undegraded":"2024-04-26T04:51:53.969660+0000","last_fullsized":"2024-04-26T04:51:53.969660+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:22:17.918565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.966621+0000","last_change":"2024-04-26T04:51:32.827791+0000","last_active":"2024-04-26T04:51:53.966621+0000","last_peered":"2024-04-26T04:51:53.966621+0000","last_clean":"2024-04-26T04:51:53.966621+0000","last_became_active":"2024-04-26T04:51:32.827416+0000","last_became_peered":"2024-04-26T04:51:32.827416+0000","last_unstale":"2024-04-26T04:51:53.966621+0000","last_undegraded":"2024-04-26T04:51:53.966621+0000","last_fullsized":"2024-04-26T04:51:53.966621+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:57:04.089430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.975702+0000","last_change":"2024-04-26T04:51:53.975702+0000","last_active":"2024-04-26T04:51:53.975702+0000","last_peered":"2024-04-26T04:51:53.975702+0000","last_clean":"2024-04-26T04:51:53.975702+0000","last_became_active":"2024-04-26T04:51:53.975351+0000","last_became_peered":"2024-04-26T04:51:53.975351+0000","last_unstale":"2024-04-26T04:51:53.975702+0000","last_undegraded":"2024-04-26T04:51:53.975702+0000","last_fullsized":"2024-04-26T04:51:53.975702+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:54:16.880776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":80,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.974369+0000","last_change":"2024-04-26T04:51:53.974369+0000","last_active":"2024-04-26T04:51:53.974369+0000","last_peered":"2024-04-26T04:51:53.974369+0000","last_clean":"2024-04-26T04:51:53.974369+0000","last_became_active":"2024-04-26T04:51:53.973909+0000","last_became_peered":"2024-04-26T04:51:53.973909+0000","last_unstale":"2024-04-26T04:51:53.974369+0000","last_undegraded":"2024-04-26T04:51:53.974369+0000","last_fullsized":"2024-04-26T04:51:53.974369+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:48:46.002054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":85,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.976299+0000","last_change":"2024-04-26T04:51:53.976299+0000","last_active":"2024-04-26T04:51:53.976299+0000","last_peered":"2024-04-26T04:51:53.976299+0000","last_clean":"2024-04-26T04:51:53.976299+0000","last_became_active":"2024-04-26T04:51:53.975944+0000","last_became_peered":"2024-04-26T04:51:53.975944+0000","last_unstale":"2024-04-26T04:51:53.976299+0000","last_undegraded":"2024-04-26T04:51:53.976299+0000","last_fullsized":"2024-04-26T04:51:53.976299+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:49:57.375693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2403,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234152,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311352,"kb_used_data":1704,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93433800,"statfs":{"total":95995035648,"available":95676211200,"internally_reserved":0,"allocated":1744896,"data_stored":1183901,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495789,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299056,"kb_used_data":1692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446096,"statfs":{"total":95995035648,"available":95688802304,"internally_reserved":0,"allocated":1732608,"data_stored":1183805,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":87,"seq":373662154781,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32792,"kb_used_data":1100,"kb_used_omap":8,"kb_used_meta":31671,"kb_avail":93712360,"statfs":{"total":95995035648,"available":95961456640,"internally_reserved":0,"allocated":1126400,"data_stored":588924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8657,"internal_metadata":32431663},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":82,"seq":352187318308,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33388,"kb_used_data":1688,"kb_used_omap":18,"kb_used_meta":31661,"kb_avail":93711764,"statfs":{"total":95995035648,"available":95960846336,"internally_reserved":0,"allocated":1728512,"data_stored":1179876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19378,"internal_metadata":32420942},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":92,"seq":395136991254,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28648,"kb_used_data":1108,"kb_used_omap":47,"kb_used_meta":27472,"kb_avail":93716504,"statfs":{"total":95995035648,"available":95965700096,"internally_reserved":0,"allocated":1134592,"data_stored":589520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49115,"internal_metadata":28131365},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":97,"seq":416611827727,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32796,"kb_used_data":1104,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93712356,"statfs":{"total":95995035648,"available":95961452544,"internally_reserved":0,"allocated":1130496,"data_stored":589006,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7697,"internal_metadata":32432623},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":102,"seq":438086664203,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28656,"kb_used_data":1124,"kb_used_omap":21,"kb_used_meta":27498,"kb_avail":93716496,"statfs":{"total":95995035648,"available":95965691904,"internally_reserved":0,"allocated":1150976,"data_stored":593798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21832,"internal_metadata":28158648},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757426,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310760,"kb_used_data":1108,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93434392,"statfs":{"total":95995035648,"available":95676817408,"internally_reserved":0,"allocated":1134592,"data_stored":589536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1064,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1339,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:52:40.158 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:52:40.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:39 smithi171 ceph-mon[90566]: pgmap v275: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:40.412 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:41.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:40 smithi156 ceph-mon[98743]: from='client.44654 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:52:41.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:40 smithi156 ceph-mon[103290]: from='client.44654 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:52:41.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:40 smithi171 ceph-mon[90566]: from='client.44654 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:52:42.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:41 smithi171 ceph-mon[90566]: pgmap v276: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:42.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:41 smithi156 ceph-mon[103290]: pgmap v276: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:42.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:41 smithi156 ceph-mon[98743]: pgmap v276: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:42.358 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:52:42.359 INFO:teuthology.orchestra.run.smithi156.stderr:dumped all 2024-04-26T04:52:43.070 INFO:teuthology.orchestra.run.smithi156.stdout:{"pg_ready":true,"pg_map":{"version":277,"stamp":"2024-04-26T04:52:42.349115+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1077448,"kb_used_data":10628,"kb_used_omap":109,"kb_used_meta":1066578,"kb_avail":748883768,"statfs":{"total":767960285184,"available":766856978432,"internally_reserved":0,"allocated":10883072,"data_stored":6498366,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":111791,"internal_metadata":1092176721},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002754"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":61,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.978555+0000","last_change":"2024-04-26T04:51:53.978555+0000","last_active":"2024-04-26T04:51:53.978555+0000","last_peered":"2024-04-26T04:51:53.978555+0000","last_clean":"2024-04-26T04:51:53.978555+0000","last_became_active":"2024-04-26T04:51:53.977847+0000","last_became_peered":"2024-04-26T04:51:53.977847+0000","last_unstale":"2024-04-26T04:51:53.978555+0000","last_undegraded":"2024-04-26T04:51:53.978555+0000","last_fullsized":"2024-04-26T04:51:53.978555+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:41:01.980360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":74,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967122+0000","last_change":"2024-04-26T04:50:57.888980+0000","last_active":"2024-04-26T04:51:53.967122+0000","last_peered":"2024-04-26T04:51:53.967122+0000","last_clean":"2024-04-26T04:51:53.967122+0000","last_became_active":"2024-04-26T04:50:57.888704+0000","last_became_peered":"2024-04-26T04:50:57.888704+0000","last_unstale":"2024-04-26T04:51:53.967122+0000","last_undegraded":"2024-04-26T04:51:53.967122+0000","last_fullsized":"2024-04-26T04:51:53.967122+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:44:48.994332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.970464+0000","last_change":"2024-04-26T04:51:32.826710+0000","last_active":"2024-04-26T04:51:53.970464+0000","last_peered":"2024-04-26T04:51:53.970464+0000","last_clean":"2024-04-26T04:51:53.970464+0000","last_became_active":"2024-04-26T04:51:32.822552+0000","last_became_peered":"2024-04-26T04:51:32.822552+0000","last_unstale":"2024-04-26T04:51:53.970464+0000","last_undegraded":"2024-04-26T04:51:53.970464+0000","last_fullsized":"2024-04-26T04:51:53.970464+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:38:35.490921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":77,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.971482+0000","last_change":"2024-04-26T04:51:32.826437+0000","last_active":"2024-04-26T04:51:53.971482+0000","last_peered":"2024-04-26T04:51:53.971482+0000","last_clean":"2024-04-26T04:51:53.971482+0000","last_became_active":"2024-04-26T04:51:32.826089+0000","last_became_peered":"2024-04-26T04:51:32.826089+0000","last_unstale":"2024-04-26T04:51:53.971482+0000","last_undegraded":"2024-04-26T04:51:53.971482+0000","last_fullsized":"2024-04-26T04:51:53.971482+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:29:20.683183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":87,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.974922+0000","last_change":"2024-04-26T04:51:53.974922+0000","last_active":"2024-04-26T04:51:53.974922+0000","last_peered":"2024-04-26T04:51:53.974922+0000","last_clean":"2024-04-26T04:51:53.974922+0000","last_became_active":"2024-04-26T04:51:53.974600+0000","last_became_peered":"2024-04-26T04:51:53.974600+0000","last_unstale":"2024-04-26T04:51:53.974922+0000","last_undegraded":"2024-04-26T04:51:53.974922+0000","last_fullsized":"2024-04-26T04:51:53.974922+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:32:49.513479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":65,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.976347+0000","last_change":"2024-04-26T04:51:53.976347+0000","last_active":"2024-04-26T04:51:53.976347+0000","last_peered":"2024-04-26T04:51:53.976347+0000","last_clean":"2024-04-26T04:51:53.976347+0000","last_became_active":"2024-04-26T04:51:53.976002+0000","last_became_peered":"2024-04-26T04:51:53.976002+0000","last_unstale":"2024-04-26T04:51:53.976347+0000","last_undegraded":"2024-04-26T04:51:53.976347+0000","last_fullsized":"2024-04-26T04:51:53.976347+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:02:30.091927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":87,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.979147+0000","last_change":"2024-04-26T04:51:53.979147+0000","last_active":"2024-04-26T04:51:53.979147+0000","last_peered":"2024-04-26T04:51:53.979147+0000","last_clean":"2024-04-26T04:51:53.979147+0000","last_became_active":"2024-04-26T04:51:53.978465+0000","last_became_peered":"2024-04-26T04:51:53.978465+0000","last_unstale":"2024-04-26T04:51:53.979147+0000","last_undegraded":"2024-04-26T04:51:53.979147+0000","last_fullsized":"2024-04-26T04:51:53.979147+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:06:00.570187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":82,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.975391+0000","last_change":"2024-04-26T04:51:53.975391+0000","last_active":"2024-04-26T04:51:53.975391+0000","last_peered":"2024-04-26T04:51:53.975391+0000","last_clean":"2024-04-26T04:51:53.975391+0000","last_became_active":"2024-04-26T04:51:53.975041+0000","last_became_peered":"2024-04-26T04:51:53.975041+0000","last_unstale":"2024-04-26T04:51:53.975391+0000","last_undegraded":"2024-04-26T04:51:53.975391+0000","last_fullsized":"2024-04-26T04:51:53.975391+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:05:37.569828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":88,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.964401+0000","last_change":"2024-04-26T04:51:32.822901+0000","last_active":"2024-04-26T04:51:53.964401+0000","last_peered":"2024-04-26T04:51:53.964401+0000","last_clean":"2024-04-26T04:51:53.964401+0000","last_became_active":"2024-04-26T04:51:32.822555+0000","last_became_peered":"2024-04-26T04:51:32.822555+0000","last_unstale":"2024-04-26T04:51:53.964401+0000","last_undegraded":"2024-04-26T04:51:53.964401+0000","last_fullsized":"2024-04-26T04:51:53.964401+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:06:22.817233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":74,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.976203+0000","last_change":"2024-04-26T04:51:53.975318+0000","last_active":"2024-04-26T04:51:53.976203+0000","last_peered":"2024-04-26T04:51:53.976203+0000","last_clean":"2024-04-26T04:51:53.976203+0000","last_became_active":"2024-04-26T04:51:53.974785+0000","last_became_peered":"2024-04-26T04:51:53.974785+0000","last_unstale":"2024-04-26T04:51:53.976203+0000","last_undegraded":"2024-04-26T04:51:53.976203+0000","last_fullsized":"2024-04-26T04:51:53.976203+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:55:04.825082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":22,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.980442+0000","last_change":"2024-04-26T04:51:53.980442+0000","last_active":"2024-04-26T04:51:53.980442+0000","last_peered":"2024-04-26T04:51:53.980442+0000","last_clean":"2024-04-26T04:51:53.980442+0000","last_became_active":"2024-04-26T04:51:53.979841+0000","last_became_peered":"2024-04-26T04:51:53.979841+0000","last_unstale":"2024-04-26T04:51:53.980442+0000","last_undegraded":"2024-04-26T04:51:53.980442+0000","last_fullsized":"2024-04-26T04:51:53.980442+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:23:38.384902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":83,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.976225+0000","last_change":"2024-04-26T04:51:53.975866+0000","last_active":"2024-04-26T04:51:53.976225+0000","last_peered":"2024-04-26T04:51:53.976225+0000","last_clean":"2024-04-26T04:51:53.976225+0000","last_became_active":"2024-04-26T04:51:53.975460+0000","last_became_peered":"2024-04-26T04:51:53.975460+0000","last_unstale":"2024-04-26T04:51:53.976225+0000","last_undegraded":"2024-04-26T04:51:53.976225+0000","last_fullsized":"2024-04-26T04:51:53.976225+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:31:34.336852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.969260+0000","last_change":"2024-04-26T04:50:57.888468+0000","last_active":"2024-04-26T04:51:52.969260+0000","last_peered":"2024-04-26T04:51:52.969260+0000","last_clean":"2024-04-26T04:51:52.969260+0000","last_became_active":"2024-04-26T04:50:57.888090+0000","last_became_peered":"2024-04-26T04:50:57.888090+0000","last_unstale":"2024-04-26T04:51:52.969260+0000","last_undegraded":"2024-04-26T04:51:52.969260+0000","last_fullsized":"2024-04-26T04:51:52.969260+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:13:00.552525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":66,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.970255+0000","last_change":"2024-04-26T04:51:32.825735+0000","last_active":"2024-04-26T04:51:53.970255+0000","last_peered":"2024-04-26T04:51:53.970255+0000","last_clean":"2024-04-26T04:51:53.970255+0000","last_became_active":"2024-04-26T04:51:32.825418+0000","last_became_peered":"2024-04-26T04:51:32.825418+0000","last_unstale":"2024-04-26T04:51:53.970255+0000","last_undegraded":"2024-04-26T04:51:53.970255+0000","last_fullsized":"2024-04-26T04:51:53.970255+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:58:10.738522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":61,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.975994+0000","last_change":"2024-04-26T04:51:53.975487+0000","last_active":"2024-04-26T04:51:53.975994+0000","last_peered":"2024-04-26T04:51:53.975994+0000","last_clean":"2024-04-26T04:51:53.975994+0000","last_became_active":"2024-04-26T04:51:53.975187+0000","last_became_peered":"2024-04-26T04:51:53.975187+0000","last_unstale":"2024-04-26T04:51:53.975994+0000","last_undegraded":"2024-04-26T04:51:53.975994+0000","last_fullsized":"2024-04-26T04:51:53.975994+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:49:35.166974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.971481+0000","last_change":"2024-04-26T04:51:32.828752+0000","last_active":"2024-04-26T04:51:53.971481+0000","last_peered":"2024-04-26T04:51:53.971481+0000","last_clean":"2024-04-26T04:51:53.971481+0000","last_became_active":"2024-04-26T04:51:32.826649+0000","last_became_peered":"2024-04-26T04:51:32.826649+0000","last_unstale":"2024-04-26T04:51:53.971481+0000","last_undegraded":"2024-04-26T04:51:53.971481+0000","last_fullsized":"2024-04-26T04:51:53.971481+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:37:03.290982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":62,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.966918+0000","last_change":"2024-04-26T04:49:48.625191+0000","last_active":"2024-04-26T04:51:53.966918+0000","last_peered":"2024-04-26T04:51:53.966918+0000","last_clean":"2024-04-26T04:51:53.966918+0000","last_became_active":"2024-04-26T04:49:48.624193+0000","last_became_peered":"2024-04-26T04:49:48.624193+0000","last_unstale":"2024-04-26T04:51:53.966918+0000","last_undegraded":"2024-04-26T04:51:53.966918+0000","last_fullsized":"2024-04-26T04:51:53.966918+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:01:56.529663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":61,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967526+0000","last_change":"2024-04-26T04:50:57.888580+0000","last_active":"2024-04-26T04:51:53.967526+0000","last_peered":"2024-04-26T04:51:53.967526+0000","last_clean":"2024-04-26T04:51:53.967526+0000","last_became_active":"2024-04-26T04:50:57.888273+0000","last_became_peered":"2024-04-26T04:50:57.888273+0000","last_unstale":"2024-04-26T04:51:53.967526+0000","last_undegraded":"2024-04-26T04:51:53.967526+0000","last_fullsized":"2024-04-26T04:51:53.967526+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:01:15.472987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":84,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.974410+0000","last_change":"2024-04-26T04:51:53.974410+0000","last_active":"2024-04-26T04:51:53.974410+0000","last_peered":"2024-04-26T04:51:53.974410+0000","last_clean":"2024-04-26T04:51:53.974410+0000","last_became_active":"2024-04-26T04:51:53.973906+0000","last_became_peered":"2024-04-26T04:51:53.973906+0000","last_unstale":"2024-04-26T04:51:53.974410+0000","last_undegraded":"2024-04-26T04:51:53.974410+0000","last_fullsized":"2024-04-26T04:51:53.974410+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:18:42.256975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":80,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967276+0000","last_change":"2024-04-26T04:50:57.888162+0000","last_active":"2024-04-26T04:51:53.967276+0000","last_peered":"2024-04-26T04:51:53.967276+0000","last_clean":"2024-04-26T04:51:53.967276+0000","last_became_active":"2024-04-26T04:50:57.887900+0000","last_became_peered":"2024-04-26T04:50:57.887900+0000","last_unstale":"2024-04-26T04:51:53.967276+0000","last_undegraded":"2024-04-26T04:51:53.967276+0000","last_fullsized":"2024-04-26T04:51:53.967276+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:59:30.563174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":80,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.975006+0000","last_change":"2024-04-26T04:51:53.975006+0000","last_active":"2024-04-26T04:51:53.975006+0000","last_peered":"2024-04-26T04:51:53.975006+0000","last_clean":"2024-04-26T04:51:53.975006+0000","last_became_active":"2024-04-26T04:51:53.974717+0000","last_became_peered":"2024-04-26T04:51:53.974717+0000","last_unstale":"2024-04-26T04:51:53.975006+0000","last_undegraded":"2024-04-26T04:51:53.975006+0000","last_fullsized":"2024-04-26T04:51:53.975006+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:57:27.538723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":84,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.970133+0000","last_change":"2024-04-26T04:50:23.338298+0000","last_active":"2024-04-26T04:51:52.970133+0000","last_peered":"2024-04-26T04:51:52.970133+0000","last_clean":"2024-04-26T04:51:52.970133+0000","last_became_active":"2024-04-26T04:50:23.337891+0000","last_became_peered":"2024-04-26T04:50:23.337891+0000","last_unstale":"2024-04-26T04:51:52.970133+0000","last_undegraded":"2024-04-26T04:51:52.970133+0000","last_fullsized":"2024-04-26T04:51:52.970133+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:01:34.076451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":63,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.977282+0000","last_change":"2024-04-26T04:51:53.977282+0000","last_active":"2024-04-26T04:51:53.977282+0000","last_peered":"2024-04-26T04:51:53.977282+0000","last_clean":"2024-04-26T04:51:53.977282+0000","last_became_active":"2024-04-26T04:51:53.976853+0000","last_became_peered":"2024-04-26T04:51:53.976853+0000","last_unstale":"2024-04-26T04:51:53.977282+0000","last_undegraded":"2024-04-26T04:51:53.977282+0000","last_fullsized":"2024-04-26T04:51:53.977282+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:26:56.909521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":32,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.970310+0000","last_change":"2024-04-26T04:51:32.829167+0000","last_active":"2024-04-26T04:51:53.970310+0000","last_peered":"2024-04-26T04:51:53.970310+0000","last_clean":"2024-04-26T04:51:53.970310+0000","last_became_active":"2024-04-26T04:51:32.827225+0000","last_became_peered":"2024-04-26T04:51:32.827225+0000","last_unstale":"2024-04-26T04:51:53.970310+0000","last_undegraded":"2024-04-26T04:51:53.970310+0000","last_fullsized":"2024-04-26T04:51:53.970310+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:00:26.343050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":442,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.970048+0000","last_change":"2024-04-26T04:49:48.625214+0000","last_active":"2024-04-26T04:51:52.970048+0000","last_peered":"2024-04-26T04:51:52.970048+0000","last_clean":"2024-04-26T04:51:52.970048+0000","last_became_active":"2024-04-26T04:49:48.624784+0000","last_became_peered":"2024-04-26T04:49:48.624784+0000","last_unstale":"2024-04-26T04:51:52.970048+0000","last_undegraded":"2024-04-26T04:51:52.970048+0000","last_fullsized":"2024-04-26T04:51:52.970048+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:36:55.213599+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:36:55.213599+0000","last_clean_scrub_stamp":"2024-04-26T04:36:55.213599+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:33:02.237214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":68,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.964303+0000","last_change":"2024-04-26T04:50:57.889224+0000","last_active":"2024-04-26T04:51:52.964303+0000","last_peered":"2024-04-26T04:51:52.964303+0000","last_clean":"2024-04-26T04:51:52.964303+0000","last_became_active":"2024-04-26T04:50:57.888860+0000","last_became_peered":"2024-04-26T04:50:57.888860+0000","last_unstale":"2024-04-26T04:51:52.964303+0000","last_undegraded":"2024-04-26T04:51:52.964303+0000","last_fullsized":"2024-04-26T04:51:52.964303+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:25:02.199086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":85,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967916+0000","last_change":"2024-04-26T04:50:23.340171+0000","last_active":"2024-04-26T04:51:53.967916+0000","last_peered":"2024-04-26T04:51:53.967916+0000","last_clean":"2024-04-26T04:51:53.967916+0000","last_became_active":"2024-04-26T04:50:23.338209+0000","last_became_peered":"2024-04-26T04:50:23.338209+0000","last_unstale":"2024-04-26T04:51:53.967916+0000","last_undegraded":"2024-04-26T04:51:53.967916+0000","last_fullsized":"2024-04-26T04:51:53.967916+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:54:07.999658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":61,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967239+0000","last_change":"2024-04-26T04:50:57.888370+0000","last_active":"2024-04-26T04:51:53.967239+0000","last_peered":"2024-04-26T04:51:53.967239+0000","last_clean":"2024-04-26T04:51:53.967239+0000","last_became_active":"2024-04-26T04:50:57.888063+0000","last_became_peered":"2024-04-26T04:50:57.888063+0000","last_unstale":"2024-04-26T04:51:53.967239+0000","last_undegraded":"2024-04-26T04:51:53.967239+0000","last_fullsized":"2024-04-26T04:51:53.967239+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:41:15.343980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":90,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.964325+0000","last_change":"2024-04-26T04:51:32.830115+0000","last_active":"2024-04-26T04:51:53.964325+0000","last_peered":"2024-04-26T04:51:53.964325+0000","last_clean":"2024-04-26T04:51:53.964325+0000","last_became_active":"2024-04-26T04:51:32.829777+0000","last_became_peered":"2024-04-26T04:51:32.829777+0000","last_unstale":"2024-04-26T04:51:53.964325+0000","last_undegraded":"2024-04-26T04:51:53.964325+0000","last_fullsized":"2024-04-26T04:51:53.964325+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:48:16.578287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967621+0000","last_change":"2024-04-26T04:51:32.825033+0000","last_active":"2024-04-26T04:51:53.967621+0000","last_peered":"2024-04-26T04:51:53.967621+0000","last_clean":"2024-04-26T04:51:53.967621+0000","last_became_active":"2024-04-26T04:51:32.824585+0000","last_became_peered":"2024-04-26T04:51:32.824585+0000","last_unstale":"2024-04-26T04:51:53.967621+0000","last_undegraded":"2024-04-26T04:51:53.967621+0000","last_fullsized":"2024-04-26T04:51:53.967621+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:04:29.404053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":57,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.980224+0000","last_change":"2024-04-26T04:51:53.980224+0000","last_active":"2024-04-26T04:51:53.980224+0000","last_peered":"2024-04-26T04:51:53.980224+0000","last_clean":"2024-04-26T04:51:53.980224+0000","last_became_active":"2024-04-26T04:51:53.979629+0000","last_became_peered":"2024-04-26T04:51:53.979629+0000","last_unstale":"2024-04-26T04:51:53.980224+0000","last_undegraded":"2024-04-26T04:51:53.980224+0000","last_fullsized":"2024-04-26T04:51:53.980224+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:02:25.756702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":68,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.965367+0000","last_change":"2024-04-26T04:50:23.336008+0000","last_active":"2024-04-26T04:51:52.965367+0000","last_peered":"2024-04-26T04:51:52.965367+0000","last_clean":"2024-04-26T04:51:52.965367+0000","last_became_active":"2024-04-26T04:50:23.335654+0000","last_became_peered":"2024-04-26T04:50:23.335654+0000","last_unstale":"2024-04-26T04:51:52.965367+0000","last_undegraded":"2024-04-26T04:51:52.965367+0000","last_fullsized":"2024-04-26T04:51:52.965367+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:50:03.658838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":52,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.970795+0000","last_change":"2024-04-26T04:51:53.970795+0000","last_active":"2024-04-26T04:51:53.970795+0000","last_peered":"2024-04-26T04:51:53.970795+0000","last_clean":"2024-04-26T04:51:53.970795+0000","last_became_active":"2024-04-26T04:51:53.970480+0000","last_became_peered":"2024-04-26T04:51:53.970480+0000","last_unstale":"2024-04-26T04:51:53.970795+0000","last_undegraded":"2024-04-26T04:51:53.970795+0000","last_fullsized":"2024-04-26T04:51:53.970795+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:16:36.478106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":80,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.964687+0000","last_change":"2024-04-26T04:51:32.827447+0000","last_active":"2024-04-26T04:51:52.964687+0000","last_peered":"2024-04-26T04:51:52.964687+0000","last_clean":"2024-04-26T04:51:52.964687+0000","last_became_active":"2024-04-26T04:51:32.826909+0000","last_became_peered":"2024-04-26T04:51:32.826909+0000","last_unstale":"2024-04-26T04:51:52.964687+0000","last_undegraded":"2024-04-26T04:51:52.964687+0000","last_fullsized":"2024-04-26T04:51:52.964687+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:08:54.742719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":91,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967429+0000","last_change":"2024-04-26T04:50:23.340374+0000","last_active":"2024-04-26T04:51:53.967429+0000","last_peered":"2024-04-26T04:51:53.967429+0000","last_clean":"2024-04-26T04:51:53.967429+0000","last_became_active":"2024-04-26T04:50:23.337021+0000","last_became_peered":"2024-04-26T04:50:23.337021+0000","last_unstale":"2024-04-26T04:51:53.967429+0000","last_undegraded":"2024-04-26T04:51:53.967429+0000","last_fullsized":"2024-04-26T04:51:53.967429+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:29.019401+0000","last_clean_scrub_stamp":"2024-04-26T04:39:29.019401+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:11:12.157249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":61,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.975022+0000","last_change":"2024-04-26T04:51:53.975022+0000","last_active":"2024-04-26T04:51:53.975022+0000","last_peered":"2024-04-26T04:51:53.975022+0000","last_clean":"2024-04-26T04:51:53.975022+0000","last_became_active":"2024-04-26T04:51:53.974709+0000","last_became_peered":"2024-04-26T04:51:53.974709+0000","last_unstale":"2024-04-26T04:51:53.975022+0000","last_undegraded":"2024-04-26T04:51:53.975022+0000","last_fullsized":"2024-04-26T04:51:53.975022+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:15:46.651017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":72,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.970349+0000","last_change":"2024-04-26T04:51:32.827981+0000","last_active":"2024-04-26T04:51:53.970349+0000","last_peered":"2024-04-26T04:51:53.970349+0000","last_clean":"2024-04-26T04:51:53.970349+0000","last_became_active":"2024-04-26T04:51:32.827654+0000","last_became_peered":"2024-04-26T04:51:32.827654+0000","last_unstale":"2024-04-26T04:51:53.970349+0000","last_undegraded":"2024-04-26T04:51:53.970349+0000","last_fullsized":"2024-04-26T04:51:53.970349+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:24:04.308350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":35,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.980388+0000","last_change":"2024-04-26T04:51:53.980388+0000","last_active":"2024-04-26T04:51:53.980388+0000","last_peered":"2024-04-26T04:51:53.980388+0000","last_clean":"2024-04-26T04:51:53.980388+0000","last_became_active":"2024-04-26T04:51:53.979784+0000","last_became_peered":"2024-04-26T04:51:53.979784+0000","last_unstale":"2024-04-26T04:51:53.980388+0000","last_undegraded":"2024-04-26T04:51:53.980388+0000","last_fullsized":"2024-04-26T04:51:53.980388+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:53:46.073564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":67,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.976228+0000","last_change":"2024-04-26T04:51:53.976228+0000","last_active":"2024-04-26T04:51:53.976228+0000","last_peered":"2024-04-26T04:51:53.976228+0000","last_clean":"2024-04-26T04:51:53.976228+0000","last_became_active":"2024-04-26T04:51:53.975872+0000","last_became_peered":"2024-04-26T04:51:53.975872+0000","last_unstale":"2024-04-26T04:51:53.976228+0000","last_undegraded":"2024-04-26T04:51:53.976228+0000","last_fullsized":"2024-04-26T04:51:53.976228+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:04:19.797962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":83,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.977758+0000","last_change":"2024-04-26T04:51:53.977758+0000","last_active":"2024-04-26T04:51:53.977758+0000","last_peered":"2024-04-26T04:51:53.977758+0000","last_clean":"2024-04-26T04:51:53.977758+0000","last_became_active":"2024-04-26T04:51:53.977117+0000","last_became_peered":"2024-04-26T04:51:53.977117+0000","last_unstale":"2024-04-26T04:51:53.977758+0000","last_undegraded":"2024-04-26T04:51:53.977758+0000","last_fullsized":"2024-04-26T04:51:53.977758+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:46:42.115079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":81,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.977884+0000","last_change":"2024-04-26T04:51:53.977884+0000","last_active":"2024-04-26T04:51:53.977884+0000","last_peered":"2024-04-26T04:51:53.977884+0000","last_clean":"2024-04-26T04:51:53.977884+0000","last_became_active":"2024-04-26T04:51:53.977583+0000","last_became_peered":"2024-04-26T04:51:53.977583+0000","last_unstale":"2024-04-26T04:51:53.977884+0000","last_undegraded":"2024-04-26T04:51:53.977884+0000","last_fullsized":"2024-04-26T04:51:53.977884+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:14:49.749925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.980287+0000","last_change":"2024-04-26T04:51:53.980287+0000","last_active":"2024-04-26T04:51:53.980287+0000","last_peered":"2024-04-26T04:51:53.980287+0000","last_clean":"2024-04-26T04:51:53.980287+0000","last_became_active":"2024-04-26T04:51:53.979720+0000","last_became_peered":"2024-04-26T04:51:53.979720+0000","last_unstale":"2024-04-26T04:51:53.980287+0000","last_undegraded":"2024-04-26T04:51:53.980287+0000","last_fullsized":"2024-04-26T04:51:53.980287+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:17:53.138239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.974532+0000","last_change":"2024-04-26T04:51:53.974532+0000","last_active":"2024-04-26T04:51:53.974532+0000","last_peered":"2024-04-26T04:51:53.974532+0000","last_clean":"2024-04-26T04:51:53.974532+0000","last_became_active":"2024-04-26T04:51:53.973947+0000","last_became_peered":"2024-04-26T04:51:53.973947+0000","last_unstale":"2024-04-26T04:51:53.974532+0000","last_undegraded":"2024-04-26T04:51:53.974532+0000","last_fullsized":"2024-04-26T04:51:53.974532+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:32:58.731977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":80,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.976335+0000","last_change":"2024-04-26T04:51:53.976012+0000","last_active":"2024-04-26T04:51:53.976335+0000","last_peered":"2024-04-26T04:51:53.976335+0000","last_clean":"2024-04-26T04:51:53.976335+0000","last_became_active":"2024-04-26T04:51:53.975654+0000","last_became_peered":"2024-04-26T04:51:53.975654+0000","last_unstale":"2024-04-26T04:51:53.976335+0000","last_undegraded":"2024-04-26T04:51:53.976335+0000","last_fullsized":"2024-04-26T04:51:53.976335+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:41:11.469823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":63,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.964436+0000","last_change":"2024-04-26T04:51:32.827338+0000","last_active":"2024-04-26T04:51:52.964436+0000","last_peered":"2024-04-26T04:51:52.964436+0000","last_clean":"2024-04-26T04:51:52.964436+0000","last_became_active":"2024-04-26T04:51:32.826934+0000","last_became_peered":"2024-04-26T04:51:32.826934+0000","last_unstale":"2024-04-26T04:51:52.964436+0000","last_undegraded":"2024-04-26T04:51:52.964436+0000","last_fullsized":"2024-04-26T04:51:52.964436+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:39:15.876554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":63,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.968009+0000","last_change":"2024-04-26T04:50:23.340925+0000","last_active":"2024-04-26T04:51:52.968009+0000","last_peered":"2024-04-26T04:51:52.968009+0000","last_clean":"2024-04-26T04:51:52.968009+0000","last_became_active":"2024-04-26T04:50:23.340605+0000","last_became_peered":"2024-04-26T04:50:23.340605+0000","last_unstale":"2024-04-26T04:51:52.968009+0000","last_undegraded":"2024-04-26T04:51:52.968009+0000","last_fullsized":"2024-04-26T04:51:52.968009+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:12:05.499630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":46,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.972685+0000","last_change":"2024-04-26T04:40:44.265256+0000","last_active":"2024-04-26T04:51:52.972685+0000","last_peered":"2024-04-26T04:51:52.972685+0000","last_clean":"2024-04-26T04:51:52.972685+0000","last_became_active":"2024-04-26T04:40:44.264777+0000","last_became_peered":"2024-04-26T04:40:44.264777+0000","last_unstale":"2024-04-26T04:51:52.972685+0000","last_undegraded":"2024-04-26T04:51:52.972685+0000","last_fullsized":"2024-04-26T04:51:52.972685+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:05:04.374938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":87,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.978735+0000","last_change":"2024-04-26T04:51:53.978735+0000","last_active":"2024-04-26T04:51:53.978735+0000","last_peered":"2024-04-26T04:51:53.978735+0000","last_clean":"2024-04-26T04:51:53.978735+0000","last_became_active":"2024-04-26T04:51:53.977433+0000","last_became_peered":"2024-04-26T04:51:53.977433+0000","last_unstale":"2024-04-26T04:51:53.978735+0000","last_undegraded":"2024-04-26T04:51:53.978735+0000","last_fullsized":"2024-04-26T04:51:53.978735+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:54:03.890519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":81,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.973442+0000","last_change":"2024-04-26T04:51:53.973442+0000","last_active":"2024-04-26T04:51:53.973442+0000","last_peered":"2024-04-26T04:51:53.973442+0000","last_clean":"2024-04-26T04:51:53.973442+0000","last_became_active":"2024-04-26T04:51:53.973098+0000","last_became_peered":"2024-04-26T04:51:53.973098+0000","last_unstale":"2024-04-26T04:51:53.973442+0000","last_undegraded":"2024-04-26T04:51:53.973442+0000","last_fullsized":"2024-04-26T04:51:53.973442+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:34:05.204900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":45,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.964231+0000","last_change":"2024-04-26T04:40:44.864902+0000","last_active":"2024-04-26T04:51:52.964231+0000","last_peered":"2024-04-26T04:51:52.964231+0000","last_clean":"2024-04-26T04:51:52.964231+0000","last_became_active":"2024-04-26T04:40:44.864603+0000","last_became_peered":"2024-04-26T04:40:44.864603+0000","last_unstale":"2024-04-26T04:51:52.964231+0000","last_undegraded":"2024-04-26T04:51:52.964231+0000","last_fullsized":"2024-04-26T04:51:52.964231+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:50:44.169231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":80,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.974364+0000","last_change":"2024-04-26T04:51:53.974364+0000","last_active":"2024-04-26T04:51:53.974364+0000","last_peered":"2024-04-26T04:51:53.974364+0000","last_clean":"2024-04-26T04:51:53.974364+0000","last_became_active":"2024-04-26T04:51:53.974031+0000","last_became_peered":"2024-04-26T04:51:53.974031+0000","last_unstale":"2024-04-26T04:51:53.974364+0000","last_undegraded":"2024-04-26T04:51:53.974364+0000","last_fullsized":"2024-04-26T04:51:53.974364+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:31:21.804373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.971391+0000","last_change":"2024-04-26T04:51:32.831989+0000","last_active":"2024-04-26T04:51:53.971391+0000","last_peered":"2024-04-26T04:51:53.971391+0000","last_clean":"2024-04-26T04:51:53.971391+0000","last_became_active":"2024-04-26T04:51:32.830987+0000","last_became_peered":"2024-04-26T04:51:32.830987+0000","last_unstale":"2024-04-26T04:51:53.971391+0000","last_undegraded":"2024-04-26T04:51:53.971391+0000","last_fullsized":"2024-04-26T04:51:53.971391+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:07:35.261462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":72,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.967042+0000","last_change":"2024-04-26T04:49:48.623488+0000","last_active":"2024-04-26T04:51:52.967042+0000","last_peered":"2024-04-26T04:51:52.967042+0000","last_clean":"2024-04-26T04:51:52.967042+0000","last_became_active":"2024-04-26T04:49:48.623070+0000","last_became_peered":"2024-04-26T04:49:48.623070+0000","last_unstale":"2024-04-26T04:51:52.967042+0000","last_undegraded":"2024-04-26T04:51:52.967042+0000","last_fullsized":"2024-04-26T04:51:52.967042+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:39:35.560811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.966562+0000","last_change":"2024-04-26T04:51:32.827894+0000","last_active":"2024-04-26T04:51:53.966562+0000","last_peered":"2024-04-26T04:51:53.966562+0000","last_clean":"2024-04-26T04:51:53.966562+0000","last_became_active":"2024-04-26T04:51:32.827492+0000","last_became_peered":"2024-04-26T04:51:32.827492+0000","last_unstale":"2024-04-26T04:51:53.966562+0000","last_undegraded":"2024-04-26T04:51:53.966562+0000","last_fullsized":"2024-04-26T04:51:53.966562+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:07:48.796063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":62,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.971090+0000","last_change":"2024-04-26T04:51:53.971090+0000","last_active":"2024-04-26T04:51:53.971090+0000","last_peered":"2024-04-26T04:51:53.971090+0000","last_clean":"2024-04-26T04:51:53.971090+0000","last_became_active":"2024-04-26T04:51:53.970806+0000","last_became_peered":"2024-04-26T04:51:53.970806+0000","last_unstale":"2024-04-26T04:51:53.971090+0000","last_undegraded":"2024-04-26T04:51:53.971090+0000","last_fullsized":"2024-04-26T04:51:53.971090+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:59:20.358643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":62,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.964605+0000","last_change":"2024-04-26T04:51:32.824417+0000","last_active":"2024-04-26T04:51:52.964605+0000","last_peered":"2024-04-26T04:51:52.964605+0000","last_clean":"2024-04-26T04:51:52.964605+0000","last_became_active":"2024-04-26T04:51:32.824013+0000","last_became_peered":"2024-04-26T04:51:32.824013+0000","last_unstale":"2024-04-26T04:51:52.964605+0000","last_undegraded":"2024-04-26T04:51:52.964605+0000","last_fullsized":"2024-04-26T04:51:52.964605+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:32:44.950039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.966601+0000","last_change":"2024-04-26T04:51:32.824357+0000","last_active":"2024-04-26T04:51:53.966601+0000","last_peered":"2024-04-26T04:51:53.966601+0000","last_clean":"2024-04-26T04:51:53.966601+0000","last_became_active":"2024-04-26T04:51:32.824033+0000","last_became_peered":"2024-04-26T04:51:32.824033+0000","last_unstale":"2024-04-26T04:51:53.966601+0000","last_undegraded":"2024-04-26T04:51:53.966601+0000","last_fullsized":"2024-04-26T04:51:53.966601+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:20:02.689022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.971378+0000","last_change":"2024-04-26T04:51:32.829027+0000","last_active":"2024-04-26T04:51:53.971378+0000","last_peered":"2024-04-26T04:51:53.971378+0000","last_clean":"2024-04-26T04:51:53.971378+0000","last_became_active":"2024-04-26T04:51:32.828359+0000","last_became_peered":"2024-04-26T04:51:32.828359+0000","last_unstale":"2024-04-26T04:51:53.971378+0000","last_undegraded":"2024-04-26T04:51:53.971378+0000","last_fullsized":"2024-04-26T04:51:53.971378+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:32:15.010228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":81,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.971881+0000","last_change":"2024-04-26T04:51:53.971881+0000","last_active":"2024-04-26T04:51:53.971881+0000","last_peered":"2024-04-26T04:51:53.971881+0000","last_clean":"2024-04-26T04:51:53.971881+0000","last_became_active":"2024-04-26T04:51:53.971550+0000","last_became_peered":"2024-04-26T04:51:53.971550+0000","last_unstale":"2024-04-26T04:51:53.971881+0000","last_undegraded":"2024-04-26T04:51:53.971881+0000","last_fullsized":"2024-04-26T04:51:53.971881+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:03:09.916815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":61,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.980248+0000","last_change":"2024-04-26T04:51:53.980248+0000","last_active":"2024-04-26T04:51:53.980248+0000","last_peered":"2024-04-26T04:51:53.980248+0000","last_clean":"2024-04-26T04:51:53.980248+0000","last_became_active":"2024-04-26T04:51:53.979699+0000","last_became_peered":"2024-04-26T04:51:53.979699+0000","last_unstale":"2024-04-26T04:51:53.980248+0000","last_undegraded":"2024-04-26T04:51:53.980248+0000","last_fullsized":"2024-04-26T04:51:53.980248+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:12:50.870138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"78'22","reported_seq":57,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.975142+0000","last_change":"2024-04-26T04:51:53.975142+0000","last_active":"2024-04-26T04:51:53.975142+0000","last_peered":"2024-04-26T04:51:53.975142+0000","last_clean":"2024-04-26T04:51:53.975142+0000","last_became_active":"2024-04-26T04:51:53.974790+0000","last_became_peered":"2024-04-26T04:51:53.974790+0000","last_unstale":"2024-04-26T04:51:53.975142+0000","last_undegraded":"2024-04-26T04:51:53.975142+0000","last_fullsized":"2024-04-26T04:51:53.975142+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:21:51.027054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":61,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967074+0000","last_change":"2024-04-26T04:50:57.888725+0000","last_active":"2024-04-26T04:51:53.967074+0000","last_peered":"2024-04-26T04:51:53.967074+0000","last_clean":"2024-04-26T04:51:53.967074+0000","last_became_active":"2024-04-26T04:50:57.887912+0000","last_became_peered":"2024-04-26T04:50:57.887912+0000","last_unstale":"2024-04-26T04:51:53.967074+0000","last_undegraded":"2024-04-26T04:51:53.967074+0000","last_fullsized":"2024-04-26T04:51:53.967074+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:03:54.843721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":82,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.974244+0000","last_change":"2024-04-26T04:51:53.974244+0000","last_active":"2024-04-26T04:51:53.974244+0000","last_peered":"2024-04-26T04:51:53.974244+0000","last_clean":"2024-04-26T04:51:53.974244+0000","last_became_active":"2024-04-26T04:51:53.973884+0000","last_became_peered":"2024-04-26T04:51:53.973884+0000","last_unstale":"2024-04-26T04:51:53.974244+0000","last_undegraded":"2024-04-26T04:51:53.974244+0000","last_fullsized":"2024-04-26T04:51:53.974244+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:52:16.159672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":32,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.970376+0000","last_change":"2024-04-26T04:51:32.830946+0000","last_active":"2024-04-26T04:51:53.970376+0000","last_peered":"2024-04-26T04:51:53.970376+0000","last_clean":"2024-04-26T04:51:53.970376+0000","last_became_active":"2024-04-26T04:51:32.829261+0000","last_became_peered":"2024-04-26T04:51:32.829261+0000","last_unstale":"2024-04-26T04:51:53.970376+0000","last_undegraded":"2024-04-26T04:51:53.970376+0000","last_fullsized":"2024-04-26T04:51:53.970376+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:11:33.805937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":63,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.964349+0000","last_change":"2024-04-26T04:50:23.339092+0000","last_active":"2024-04-26T04:51:52.964349+0000","last_peered":"2024-04-26T04:51:52.964349+0000","last_clean":"2024-04-26T04:51:52.964349+0000","last_became_active":"2024-04-26T04:50:23.338761+0000","last_became_peered":"2024-04-26T04:50:23.338761+0000","last_unstale":"2024-04-26T04:51:52.964349+0000","last_undegraded":"2024-04-26T04:51:52.964349+0000","last_fullsized":"2024-04-26T04:51:52.964349+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:55:21.983639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.969495+0000","last_change":"2024-04-26T04:51:32.829000+0000","last_active":"2024-04-26T04:51:53.969495+0000","last_peered":"2024-04-26T04:51:53.969495+0000","last_clean":"2024-04-26T04:51:53.969495+0000","last_became_active":"2024-04-26T04:51:32.828549+0000","last_became_peered":"2024-04-26T04:51:32.828549+0000","last_unstale":"2024-04-26T04:51:53.969495+0000","last_undegraded":"2024-04-26T04:51:53.969495+0000","last_fullsized":"2024-04-26T04:51:53.969495+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:43:34.903419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.967258+0000","last_change":"2024-04-26T04:51:32.827496+0000","last_active":"2024-04-26T04:51:53.967258+0000","last_peered":"2024-04-26T04:51:53.967258+0000","last_clean":"2024-04-26T04:51:53.967258+0000","last_became_active":"2024-04-26T04:51:32.827199+0000","last_became_peered":"2024-04-26T04:51:32.827199+0000","last_unstale":"2024-04-26T04:51:53.967258+0000","last_undegraded":"2024-04-26T04:51:53.967258+0000","last_fullsized":"2024-04-26T04:51:53.967258+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:00:20.621104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":63,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-04-26T04:51:52.969337+0000","last_change":"2024-04-26T04:50:23.336719+0000","last_active":"2024-04-26T04:51:52.969337+0000","last_peered":"2024-04-26T04:51:52.969337+0000","last_clean":"2024-04-26T04:51:52.969337+0000","last_became_active":"2024-04-26T04:50:23.336382+0000","last_became_peered":"2024-04-26T04:50:23.336382+0000","last_unstale":"2024-04-26T04:51:52.969337+0000","last_undegraded":"2024-04-26T04:51:52.969337+0000","last_fullsized":"2024-04-26T04:51:52.969337+0000","mapping_epoch":87,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":88,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:36:29.868029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.969660+0000","last_change":"2024-04-26T04:51:32.825815+0000","last_active":"2024-04-26T04:51:53.969660+0000","last_peered":"2024-04-26T04:51:53.969660+0000","last_clean":"2024-04-26T04:51:53.969660+0000","last_became_active":"2024-04-26T04:51:32.825485+0000","last_became_peered":"2024-04-26T04:51:32.825485+0000","last_unstale":"2024-04-26T04:51:53.969660+0000","last_undegraded":"2024-04-26T04:51:53.969660+0000","last_fullsized":"2024-04-26T04:51:53.969660+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:22:17.918565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.966621+0000","last_change":"2024-04-26T04:51:32.827791+0000","last_active":"2024-04-26T04:51:53.966621+0000","last_peered":"2024-04-26T04:51:53.966621+0000","last_clean":"2024-04-26T04:51:53.966621+0000","last_became_active":"2024-04-26T04:51:32.827416+0000","last_became_peered":"2024-04-26T04:51:32.827416+0000","last_unstale":"2024-04-26T04:51:53.966621+0000","last_undegraded":"2024-04-26T04:51:53.966621+0000","last_fullsized":"2024-04-26T04:51:53.966621+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:57:04.089430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.975702+0000","last_change":"2024-04-26T04:51:53.975702+0000","last_active":"2024-04-26T04:51:53.975702+0000","last_peered":"2024-04-26T04:51:53.975702+0000","last_clean":"2024-04-26T04:51:53.975702+0000","last_became_active":"2024-04-26T04:51:53.975351+0000","last_became_peered":"2024-04-26T04:51:53.975351+0000","last_unstale":"2024-04-26T04:51:53.975702+0000","last_undegraded":"2024-04-26T04:51:53.975702+0000","last_fullsized":"2024-04-26T04:51:53.975702+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:54:16.880776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":80,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.974369+0000","last_change":"2024-04-26T04:51:53.974369+0000","last_active":"2024-04-26T04:51:53.974369+0000","last_peered":"2024-04-26T04:51:53.974369+0000","last_clean":"2024-04-26T04:51:53.974369+0000","last_became_active":"2024-04-26T04:51:53.973909+0000","last_became_peered":"2024-04-26T04:51:53.973909+0000","last_unstale":"2024-04-26T04:51:53.974369+0000","last_undegraded":"2024-04-26T04:51:53.974369+0000","last_fullsized":"2024-04-26T04:51:53.974369+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:44.250635+0000","last_clean_scrub_stamp":"2024-04-26T04:40:44.250635+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:48:46.002054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":85,"reported_epoch":103,"state":"active+clean","last_fresh":"2024-04-26T04:51:53.976299+0000","last_change":"2024-04-26T04:51:53.976299+0000","last_active":"2024-04-26T04:51:53.976299+0000","last_peered":"2024-04-26T04:51:53.976299+0000","last_clean":"2024-04-26T04:51:53.976299+0000","last_became_active":"2024-04-26T04:51:53.975944+0000","last_became_peered":"2024-04-26T04:51:53.975944+0000","last_unstale":"2024-04-26T04:51:53.976299+0000","last_undegraded":"2024-04-26T04:51:53.976299+0000","last_fullsized":"2024-04-26T04:51:53.976299+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:40:43.243979+0000","last_clean_scrub_stamp":"2024-04-26T04:40:43.243979+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:49:57.375693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2403,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234153,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311352,"kb_used_data":1704,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93433800,"statfs":{"total":95995035648,"available":95676211200,"internally_reserved":0,"allocated":1744896,"data_stored":1183901,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495790,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299056,"kb_used_data":1692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446096,"statfs":{"total":95995035648,"available":95688802304,"internally_reserved":0,"allocated":1732608,"data_stored":1183805,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":87,"seq":373662154782,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32792,"kb_used_data":1100,"kb_used_omap":8,"kb_used_meta":31671,"kb_avail":93712360,"statfs":{"total":95995035648,"available":95961456640,"internally_reserved":0,"allocated":1126400,"data_stored":588924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8657,"internal_metadata":32431663},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":82,"seq":352187318309,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33388,"kb_used_data":1688,"kb_used_omap":18,"kb_used_meta":31661,"kb_avail":93711764,"statfs":{"total":95995035648,"available":95960846336,"internally_reserved":0,"allocated":1728512,"data_stored":1179876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19378,"internal_metadata":32420942},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":92,"seq":395136991255,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28648,"kb_used_data":1108,"kb_used_omap":47,"kb_used_meta":27472,"kb_avail":93716504,"statfs":{"total":95995035648,"available":95965700096,"internally_reserved":0,"allocated":1134592,"data_stored":589520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49115,"internal_metadata":28131365},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":97,"seq":416611827728,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32796,"kb_used_data":1104,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93712356,"statfs":{"total":95995035648,"available":95961452544,"internally_reserved":0,"allocated":1130496,"data_stored":589006,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7697,"internal_metadata":32432623},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":102,"seq":438086664204,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28656,"kb_used_data":1124,"kb_used_omap":21,"kb_used_meta":27498,"kb_avail":93716496,"statfs":{"total":95995035648,"available":95965691904,"internally_reserved":0,"allocated":1150976,"data_stored":593798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":21832,"internal_metadata":28158648},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757427,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310760,"kb_used_data":1108,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93434392,"statfs":{"total":95995035648,"available":95676817408,"internally_reserved":0,"allocated":1134592,"data_stored":589536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1064,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1339,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:52:43.073 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-26T04:52:43.073 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-26T04:52:43.073 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-26T04:52:43.073 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:52:43.338 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:44.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:43 smithi156 ceph-mon[98743]: pgmap v277: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:44.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:43 smithi156 ceph-mon[98743]: from='client.44660 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:52:44.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:43 smithi156 ceph-mon[103290]: pgmap v277: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:44.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:43 smithi156 ceph-mon[103290]: from='client.44660 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:52:44.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:43 smithi171 ceph-mon[90566]: pgmap v277: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:44.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:43 smithi171 ceph-mon[90566]: from='client.44660 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:52:44.823 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:52:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:52:44] "GET /metrics HTTP/1.1" 200 33899 "" "Prometheus/2.43.0" 2024-04-26T04:52:45.315 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:52:45.315 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:52:46.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:45 smithi171 ceph-mon[90566]: pgmap v278: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:46.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:45 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3930269849' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:52:46.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:45 smithi156 ceph-mon[103290]: pgmap v278: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:46.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:45 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3930269849' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:52:46.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:45 smithi156 ceph-mon[98743]: pgmap v278: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:46.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:45 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3930269849' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:52:48.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:47 smithi171 ceph-mon[90566]: pgmap v279: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:48.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:47 smithi156 ceph-mon[103290]: pgmap v279: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:48.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:47 smithi156 ceph-mon[98743]: pgmap v279: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:48.944 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:52:49.211 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:50.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:49 smithi171 ceph-mon[90566]: pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:50.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:49 smithi156 ceph-mon[98743]: pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:50.385 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:49 smithi156 ceph-mon[103290]: pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:51.002 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:50 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:51.002 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:50 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:51.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:50 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:51.253 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:52:51.253 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:52:52.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:51 smithi171 ceph-mon[90566]: pgmap v281: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:52.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:51 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/4258745061' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:52:52.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:51 smithi156 ceph-mon[103290]: pgmap v281: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:52.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:51 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/4258745061' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:52:52.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:51 smithi156 ceph-mon[98743]: pgmap v281: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:52.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:51 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/4258745061' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:52:54.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:53 smithi171 ceph-mon[90566]: pgmap v282: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:54.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:53 smithi156 ceph-mon[98743]: pgmap v282: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:54.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:53 smithi156 ceph-mon[103290]: pgmap v282: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:54.966 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:52:54.992 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:52:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:52:54] "GET /metrics HTTP/1.1" 200 33896 "" "Prometheus/2.43.0" 2024-04-26T04:52:55.228 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:52:56.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:55 smithi171 ceph-mon[90566]: pgmap v283: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:56.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:55 smithi156 ceph-mon[103290]: pgmap v283: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:56.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:55 smithi156 ceph-mon[98743]: pgmap v283: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:57.066 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:56 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:57.067 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:56 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:57.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:56 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:57.328 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:52:57.328 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:52:58.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:57 smithi156 ceph-mon[98743]: pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:58.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:57 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1276165005' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:52:58.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:57 smithi156 ceph-mon[103290]: pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:58.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:57 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1276165005' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:52:58.231 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:57 smithi171 ceph-mon[90566]: pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:52:58.231 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:57 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1276165005' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:00.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:52:59 smithi171 ceph-mon[90566]: pgmap v285: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:00.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:52:59 smithi156 ceph-mon[98743]: pgmap v285: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:00.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:52:59 smithi156 ceph-mon[103290]: pgmap v285: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:01.588 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:53:01.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:01 smithi171 ceph-mon[90566]: pgmap v286: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:01.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:01 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:53:01.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:01 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:53:01.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:01 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:53:01.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:01 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:53:01.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:01 smithi156 ceph-mon[98743]: pgmap v286: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:01.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:01 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:53:01.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:01 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:53:01.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:01 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:53:01.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:01 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:53:01.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:01 smithi156 ceph-mon[103290]: pgmap v286: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:01.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:01 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:53:01.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:01 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:53:01.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:01 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:53:01.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:01 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:53:01.849 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:02.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:02 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:02.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:02 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:02.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:02 smithi156 ceph-mon[98743]: pgmap v287: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:02.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:02 smithi156 ceph-mon[98743]: pgmap v288: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:02.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:02 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:53:02.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:02 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:02.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:02 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:02.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:02 smithi156 ceph-mon[103290]: pgmap v287: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:02.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:02 smithi156 ceph-mon[103290]: pgmap v288: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:02.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:02 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:53:02.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:02 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:02.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:02 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:02.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:02 smithi171 ceph-mon[90566]: pgmap v287: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:02.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:02 smithi171 ceph-mon[90566]: pgmap v288: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:02.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:02 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:53:03.779 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:53:03.779 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:53:04.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:04 smithi171 ceph-mon[90566]: pgmap v289: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:04.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:04 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1204001953' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:05.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:04 smithi156 ceph-mon[98743]: pgmap v289: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:05.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:04 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1204001953' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:05.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:04 smithi156 ceph-mon[103290]: pgmap v289: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:05.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:04 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1204001953' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:05.094 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:53:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:53:04] "GET /metrics HTTP/1.1" 200 33896 "" "Prometheus/2.43.0" 2024-04-26T04:53:05.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:05 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:06.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:05 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:06.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:06.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:06 smithi171 ceph-mon[90566]: pgmap v290: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:07.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:06 smithi156 ceph-mon[103290]: pgmap v290: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:07.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:06 smithi156 ceph-mon[98743]: pgmap v290: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:07.430 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:53:07.695 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:08.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:08 smithi156 ceph-mon[98743]: pgmap v291: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:08.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:08 smithi156 ceph-mon[103290]: pgmap v291: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:08.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:08 smithi171 ceph-mon[90566]: pgmap v291: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:09.637 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:53:09.637 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:53:09.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:09 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2486379198' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:10.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:09 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2486379198' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:10.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:09 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2486379198' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:10.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:10 smithi171 ceph-mon[90566]: pgmap v292: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:11.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:10 smithi156 ceph-mon[98743]: pgmap v292: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:11.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:10 smithi156 ceph-mon[103290]: pgmap v292: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:12.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:12 smithi171 ceph-mon[90566]: pgmap v293: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:13.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:12 smithi156 ceph-mon[98743]: pgmap v293: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:13.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:12 smithi156 ceph-mon[103290]: pgmap v293: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:13.248 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:53:13.514 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:14.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:14 smithi171 ceph-mon[90566]: pgmap v294: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:15.059 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:14 smithi156 ceph-mon[98743]: pgmap v294: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:15.059 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:53:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:53:14] "GET /metrics HTTP/1.1" 200 33897 "" "Prometheus/2.43.0" 2024-04-26T04:53:15.059 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:14 smithi156 ceph-mon[103290]: pgmap v294: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:15.568 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:53:15.568 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:53:15.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:15 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/85130294' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:15.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:15 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/85130294' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:15.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:15 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/85130294' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:16.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:16 smithi171 ceph-mon[90566]: pgmap v295: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:17.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:16 smithi156 ceph-mon[103290]: pgmap v295: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:17.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:16 smithi156 ceph-mon[98743]: pgmap v295: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:18.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:18 smithi171 ceph-mon[90566]: pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:19.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:18 smithi156 ceph-mon[98743]: pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:19.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:18 smithi156 ceph-mon[103290]: pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:19.198 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:53:19.462 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:20.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:20 smithi171 ceph-mon[90566]: pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:20.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:21.046 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:20 smithi156 ceph-mon[103290]: pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:21.046 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:21.046 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:20 smithi156 ceph-mon[98743]: pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:21.046 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:21.542 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:53:21.542 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:53:21.792 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:21 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3743260309' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:21.792 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:21 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3743260309' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:21.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:21 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3743260309' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:23.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:22 smithi156 ceph-mon[103290]: pgmap v298: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:23.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:22 smithi156 ceph-mon[98743]: pgmap v298: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:23.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:22 smithi171 ceph-mon[90566]: pgmap v298: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:25.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:24 smithi156 ceph-mon[98743]: pgmap v299: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:25.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:53:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:53:24] "GET /metrics HTTP/1.1" 200 33902 "" "Prometheus/2.43.0" 2024-04-26T04:53:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:24 smithi156 ceph-mon[103290]: pgmap v299: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:25.198 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:53:25.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:24 smithi171 ceph-mon[90566]: pgmap v299: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:25.463 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:26.980 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:26 smithi156 ceph-mon[98743]: pgmap v300: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:26.981 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:26 smithi156 ceph-mon[103290]: pgmap v300: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:27.052 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:26 smithi171 ceph-mon[90566]: pgmap v300: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:27.492 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:53:27.492 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:53:28.067 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:27 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2230633675' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:28.089 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:27 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2230633675' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:28.089 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:27 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2230633675' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:29.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:28 smithi156 ceph-mon[98743]: pgmap v301: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:29.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:28 smithi156 ceph-mon[103290]: pgmap v301: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:29.244 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:28 smithi171 ceph-mon[90566]: pgmap v301: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:31.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:30 smithi156 ceph-mon[98743]: pgmap v302: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:31.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:30 smithi156 ceph-mon[103290]: pgmap v302: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:31.115 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:53:31.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:30 smithi171 ceph-mon[90566]: pgmap v302: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:31.381 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:32.937 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:32 smithi156 ceph-mon[98743]: pgmap v303: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:32.937 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:32 smithi156 ceph-mon[103290]: pgmap v303: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:33.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:32 smithi171 ceph-mon[90566]: pgmap v303: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:33.410 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:53:33.410 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:53:34.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:33 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3410806636' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:34.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:33 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3410806636' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:34.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:33 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3410806636' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:35.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:34 smithi156 ceph-mon[98743]: pgmap v304: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:35.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:53:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:53:34] "GET /metrics HTTP/1.1" 200 33902 "" "Prometheus/2.43.0" 2024-04-26T04:53:35.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:34 smithi156 ceph-mon[103290]: pgmap v304: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:35.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:34 smithi171 ceph-mon[90566]: pgmap v304: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:36.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:36.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:36.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:35 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:37.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:36 smithi156 ceph-mon[98743]: pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:37.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:36 smithi156 ceph-mon[103290]: pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:37.165 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:53:37.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:36 smithi171 ceph-mon[90566]: pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:37.428 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:38.910 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:38 smithi156 ceph-mon[98743]: pgmap v306: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:38.910 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:38 smithi156 ceph-mon[103290]: pgmap v306: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:39.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:38 smithi171 ceph-mon[90566]: pgmap v306: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:39.411 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:53:39.411 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:53:40.060 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:39 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2599756007' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:40.060 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:39 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2599756007' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:40.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:39 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2599756007' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:41.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:40 smithi156 ceph-mon[103290]: pgmap v307: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:41.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:40 smithi156 ceph-mon[98743]: pgmap v307: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:41.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:40 smithi171 ceph-mon[90566]: pgmap v307: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:43.086 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:53:43.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:42 smithi156 ceph-mon[98743]: pgmap v308: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:43.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:42 smithi156 ceph-mon[103290]: pgmap v308: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:43.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:42 smithi171 ceph-mon[90566]: pgmap v308: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:43.355 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:44.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:44 smithi156 ceph-mon[98743]: pgmap v309: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:44.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:53:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:53:44] "GET /metrics HTTP/1.1" 200 33894 "" "Prometheus/2.43.0" 2024-04-26T04:53:44.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:44 smithi156 ceph-mon[103290]: pgmap v309: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:45.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:44 smithi171 ceph-mon[90566]: pgmap v309: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:45.361 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:53:45.361 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:53:45.950 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:45 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1799389044' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:45.950 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:45 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1799389044' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:46.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:45 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1799389044' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:47.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:46 smithi156 ceph-mon[98743]: pgmap v310: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:47.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:46 smithi156 ceph-mon[103290]: pgmap v310: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:47.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:46 smithi171 ceph-mon[90566]: pgmap v310: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:48.975 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:53:49.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:48 smithi156 ceph-mon[98743]: pgmap v311: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:49.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:48 smithi156 ceph-mon[103290]: pgmap v311: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:49.241 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:49.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:48 smithi171 ceph-mon[90566]: pgmap v311: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:50.902 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:50 smithi156 ceph-mon[98743]: pgmap v312: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:50.903 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:50 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:50.903 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:50 smithi156 ceph-mon[103290]: pgmap v312: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:50.903 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:50 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:51.144 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:53:51.145 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:53:51.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:50 smithi171 ceph-mon[90566]: pgmap v312: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:51.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:50 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:52.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:51 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3288583630' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:52.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:51 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3288583630' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:52.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:51 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3288583630' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:53.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:52 smithi156 ceph-mon[103290]: pgmap v313: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:53.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:52 smithi156 ceph-mon[98743]: pgmap v313: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:53.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:52 smithi171 ceph-mon[90566]: pgmap v313: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:54.756 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:53:55.026 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:55.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:54 smithi156 ceph-mon[98743]: pgmap v314: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:55.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:53:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:53:54] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.43.0" 2024-04-26T04:53:55.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:54 smithi156 ceph-mon[103290]: pgmap v314: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:55.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:54 smithi171 ceph-mon[90566]: pgmap v314: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:56.957 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:53:56.957 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:53:57.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:56 smithi171 ceph-mon[90566]: pgmap v315: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:57.261 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:56 smithi156 ceph-mon[98743]: pgmap v315: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:57.262 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:56 smithi156 ceph-mon[103290]: pgmap v315: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:58.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:57 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/425260307' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:58.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:57 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/425260307' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:58.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:57 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/425260307' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:53:59.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:53:58 smithi171 ceph-mon[90566]: pgmap v316: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:59.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:53:58 smithi156 ceph-mon[103290]: pgmap v316: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:53:59.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:53:58 smithi156 ceph-mon[98743]: pgmap v316: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:00.605 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:54:00.869 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:01.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:00 smithi171 ceph-mon[90566]: pgmap v317: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:01.290 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:00 smithi156 ceph-mon[98743]: pgmap v317: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:01.290 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:00 smithi156 ceph-mon[103290]: pgmap v317: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:02.069 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:01 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:02.069 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:01 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:02.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:01 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:02.912 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:54:02.912 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:54:03.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:02 smithi171 ceph-mon[90566]: pgmap v318: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:03.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:02 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/706469026' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:03.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:02 smithi156 ceph-mon[98743]: pgmap v318: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:03.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:02 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/706469026' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:03.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:02 smithi156 ceph-mon[103290]: pgmap v318: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:03.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:02 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/706469026' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:04.950 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:54:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:54:04] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.43.0" 2024-04-26T04:54:05.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:04 smithi171 ceph-mon[90566]: pgmap v319: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:05.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:04 smithi156 ceph-mon[98743]: pgmap v319: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:05.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:04 smithi156 ceph-mon[103290]: pgmap v319: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:06.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:05 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:06.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:05 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:54:06.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:05 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:54:06.253 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:06.254 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:54:06.254 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:54:06.254 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:05 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:06.254 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:05 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:54:06.254 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:05 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:54:07.171 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:54:07.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:06 smithi171 ceph-mon[90566]: pgmap v320: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:07.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:06 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:54:07.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:06 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:54:07.269 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:06 smithi156 ceph-mon[98743]: pgmap v320: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:07.270 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:06 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:54:07.270 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:06 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:54:07.270 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:06 smithi156 ceph-mon[103290]: pgmap v320: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:07.270 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:06 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:54:07.270 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:06 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:54:07.499 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:08.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:07 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:08.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:07 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:08.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:07 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:54:08.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:07 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:08.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:07 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:08.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:07 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:54:08.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:07 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:08.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:07 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:08.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:07 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:54:09.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:08 smithi171 ceph-mon[90566]: pgmap v321: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:09.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:08 smithi171 ceph-mon[90566]: pgmap v322: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:09.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:08 smithi156 ceph-mon[98743]: pgmap v321: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:09.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:08 smithi156 ceph-mon[98743]: pgmap v322: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:09.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:08 smithi156 ceph-mon[103290]: pgmap v321: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:09.344 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:08 smithi156 ceph-mon[103290]: pgmap v322: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:09.429 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:54:09.429 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:54:10.083 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:09 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3176252589' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:10.083 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:09 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3176252589' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:10.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:09 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3176252589' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:11.244 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:10 smithi156 ceph-mon[98743]: pgmap v323: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:11.245 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:10 smithi156 ceph-mon[103290]: pgmap v323: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:11.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:10 smithi171 ceph-mon[90566]: pgmap v323: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:13.108 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:54:13.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:12 smithi171 ceph-mon[90566]: pgmap v324: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:13.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:12 smithi156 ceph-mon[98743]: pgmap v324: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:13.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:12 smithi156 ceph-mon[103290]: pgmap v324: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:13.376 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:14.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:54:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:54:14] "GET /metrics HTTP/1.1" 200 33895 "" "Prometheus/2.43.0" 2024-04-26T04:54:15.118 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:14 smithi156 ceph-mon[103290]: pgmap v325: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:15.118 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:14 smithi156 ceph-mon[98743]: pgmap v325: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:15.378 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:54:15.378 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:54:15.456 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:14 smithi171 ceph-mon[90566]: pgmap v325: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:16.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:15 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1637874833' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:16.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:15 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1637874833' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:16.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:15 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1637874833' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:17.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:16 smithi171 ceph-mon[90566]: pgmap v326: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:17.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:16 smithi156 ceph-mon[103290]: pgmap v326: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:17.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:16 smithi156 ceph-mon[98743]: pgmap v326: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:19.022 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:54:19.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:18 smithi171 ceph-mon[90566]: pgmap v327: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:19.289 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:19.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:18 smithi156 ceph-mon[98743]: pgmap v327: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:19.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:18 smithi156 ceph-mon[103290]: pgmap v327: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:20.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:19 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:20.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:19 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:20.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:19 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:21.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:20 smithi156 ceph-mon[103290]: pgmap v328: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:21.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:20 smithi156 ceph-mon[98743]: pgmap v328: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:21.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:20 smithi171 ceph-mon[90566]: pgmap v328: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:21.304 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:54:21.304 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:54:22.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:21 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2208523845' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:22.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:21 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2208523845' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:22.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:21 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2208523845' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:23.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:22 smithi171 ceph-mon[90566]: pgmap v329: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:23.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:22 smithi156 ceph-mon[98743]: pgmap v329: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:23.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:22 smithi156 ceph-mon[103290]: pgmap v329: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:24.997 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:54:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:54:24] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.43.0" 2024-04-26T04:54:25.030 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:54:25.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:24 smithi171 ceph-mon[90566]: pgmap v330: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:25.296 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:25.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:24 smithi156 ceph-mon[103290]: pgmap v330: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:25.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:24 smithi156 ceph-mon[98743]: pgmap v330: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:27.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:27 smithi156 ceph-mon[98743]: pgmap v331: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:27.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:27 smithi156 ceph-mon[103290]: pgmap v331: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:27.309 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:54:27.310 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:54:27.322 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:27 smithi171 ceph-mon[90566]: pgmap v331: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:28.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:28 smithi156 ceph-mon[103290]: pgmap v332: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:28.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:28 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2506959189' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:28.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:28 smithi156 ceph-mon[98743]: pgmap v332: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:28.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:28 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2506959189' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:28.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:28 smithi171 ceph-mon[90566]: pgmap v332: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:28.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:28 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2506959189' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:30.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:30 smithi171 ceph-mon[90566]: pgmap v333: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:30.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:30 smithi156 ceph-mon[103290]: pgmap v333: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:30.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:30 smithi156 ceph-mon[98743]: pgmap v333: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:30.989 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:54:31.256 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:32.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:32 smithi171 ceph-mon[90566]: pgmap v334: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:32.578 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:32 smithi156 ceph-mon[103290]: pgmap v334: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:32.578 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:32 smithi156 ceph-mon[98743]: pgmap v334: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:33.399 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:54:33.399 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:54:34.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:34 smithi156 ceph-mon[98743]: pgmap v335: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:34.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:34 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/4229670718' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:34.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:34 smithi156 ceph-mon[103290]: pgmap v335: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:34.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:34 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/4229670718' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:34.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:34 smithi171 ceph-mon[90566]: pgmap v335: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:34.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:34 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/4229670718' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:35.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:54:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:54:34] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.43.0" 2024-04-26T04:54:35.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:35 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:35.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:35.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:36.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:36 smithi171 ceph-mon[90566]: pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:36.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:36 smithi156 ceph-mon[98743]: pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:36.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:36 smithi156 ceph-mon[103290]: pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:37.046 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:54:37.314 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:38.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:38 smithi171 ceph-mon[90566]: pgmap v337: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:38.525 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:38 smithi156 ceph-mon[103290]: pgmap v337: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:38.525 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:38 smithi156 ceph-mon[98743]: pgmap v337: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:39.298 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:54:39.298 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:54:40.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:40 smithi156 ceph-mon[103290]: pgmap v338: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:40.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:40 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/862588332' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:40.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:40 smithi156 ceph-mon[98743]: pgmap v338: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:40.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:40 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/862588332' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:40.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:40 smithi171 ceph-mon[90566]: pgmap v338: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:40.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:40 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/862588332' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:42.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:42 smithi171 ceph-mon[90566]: pgmap v339: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:42.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:42 smithi156 ceph-mon[103290]: pgmap v339: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:42.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:42 smithi156 ceph-mon[98743]: pgmap v339: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:43.111 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:54:43.377 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:44.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:44 smithi171 ceph-mon[90566]: pgmap v340: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:44.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:44 smithi156 ceph-mon[98743]: pgmap v340: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:44.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:44 smithi156 ceph-mon[103290]: pgmap v340: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:44.860 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:54:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:54:44] "GET /metrics HTTP/1.1" 200 33897 "" "Prometheus/2.43.0" 2024-04-26T04:54:45.347 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:54:45.348 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:54:46.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:46 smithi156 ceph-mon[98743]: pgmap v341: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:46.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:46 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2031392928' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:46.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:46 smithi156 ceph-mon[103290]: pgmap v341: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:46.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:46 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2031392928' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:46.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:46 smithi171 ceph-mon[90566]: pgmap v341: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:46.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:46 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2031392928' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:48.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:48 smithi171 ceph-mon[90566]: pgmap v342: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:48.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:48 smithi156 ceph-mon[98743]: pgmap v342: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:48.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:48 smithi156 ceph-mon[103290]: pgmap v342: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:49.024 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:54:49.292 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:50.521 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:50 smithi156 ceph-mon[103290]: pgmap v343: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:50.521 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:50 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:50.521 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:50 smithi156 ceph-mon[98743]: pgmap v343: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:50.521 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:50 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:50.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:50 smithi171 ceph-mon[90566]: pgmap v343: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:50.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:50 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:51.243 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:54:51.243 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:54:51.554 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:51 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1945294137' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:51.554 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:51 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1945294137' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:51.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:51 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1945294137' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:52.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:52 smithi156 ceph-mon[98743]: pgmap v344: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:52.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:52 smithi156 ceph-mon[103290]: pgmap v344: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:52.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:52 smithi171 ceph-mon[90566]: pgmap v344: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:54.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:54 smithi156 ceph-mon[98743]: pgmap v345: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:54.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:54 smithi156 ceph-mon[103290]: pgmap v345: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:54.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:54 smithi171 ceph-mon[90566]: pgmap v345: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:54.897 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:54:55.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:54:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:54:54] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.43.0" 2024-04-26T04:54:55.164 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:56.521 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:56 smithi156 ceph-mon[98743]: pgmap v346: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:56.521 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:56 smithi156 ceph-mon[103290]: pgmap v346: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:56.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:56 smithi171 ceph-mon[90566]: pgmap v346: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:57.034 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:54:57.034 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:54:57.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:57 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3253775132' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:57.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:57 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3253775132' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:57.651 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:57 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3253775132' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:54:58.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:54:58 smithi156 ceph-mon[98743]: pgmap v347: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:58.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:54:58 smithi156 ceph-mon[103290]: pgmap v347: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:54:58.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:54:58 smithi171 ceph-mon[90566]: pgmap v347: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:00.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:00 smithi156 ceph-mon[98743]: pgmap v348: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:00.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:00 smithi156 ceph-mon[103290]: pgmap v348: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:00.677 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:55:00.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:00 smithi171 ceph-mon[90566]: pgmap v348: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:00.946 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:02.592 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:02 smithi156 ceph-mon[103290]: pgmap v349: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:02.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:02 smithi156 ceph-mon[98743]: pgmap v349: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:02.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:02 smithi171 ceph-mon[90566]: pgmap v349: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:03.084 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:55:03.084 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:55:03.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:03 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/712514784' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:03.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:03 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/712514784' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:03.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:03 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/712514784' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:04.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:04 smithi156 ceph-mon[98743]: pgmap v350: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:04.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:04 smithi156 ceph-mon[103290]: pgmap v350: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:04.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:04 smithi171 ceph-mon[90566]: pgmap v350: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:05.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:55:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:55:04] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.43.0" 2024-04-26T04:55:05.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:05.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:05 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:05.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:05 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:06.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:06 smithi156 ceph-mon[98743]: pgmap v351: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:06.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:06 smithi156 ceph-mon[103290]: pgmap v351: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:06.648 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:55:06.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:06 smithi171 ceph-mon[90566]: pgmap v351: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:06.914 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:07.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:07 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:07.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:07 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:07.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:07 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:08.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:08 smithi171 ceph-mon[90566]: pgmap v352: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:08.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:08 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:09.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:08 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:09.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:08 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:09.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:08 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:09.033 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:55:09.033 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:55:09.033 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:08 smithi156 ceph-mon[103290]: pgmap v352: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:09.033 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:08 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:09.033 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:08 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:09.034 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:08 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:09.034 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:08 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:09.034 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:08 smithi156 ceph-mon[98743]: pgmap v352: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:09.034 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:08 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:09.034 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:08 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:09.034 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:08 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:09.034 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:08 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:09.795 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:09 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3782178129' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:09.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:09 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3782178129' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:10.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:09 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3782178129' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:11.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:10 smithi156 ceph-mon[98743]: pgmap v353: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:11.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:10 smithi156 ceph-mon[103290]: pgmap v353: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:11.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:10 smithi171 ceph-mon[90566]: pgmap v353: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:12.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:12 smithi156 ceph-mon[103290]: pgmap v354: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:12.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:12 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:12.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:12 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:12.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:12 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:12.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:12 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:12.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:12 smithi156 ceph-mon[98743]: pgmap v354: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:12.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:12 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:12.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:12 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:12.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:12 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:12.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:12 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:12.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:12 smithi171 ceph-mon[90566]: pgmap v354: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:12.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:12 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:12.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:12 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:12.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:12 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:13.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:12 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:13.297 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:55:13.560 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:13.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:13 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:13.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:13 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:14.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:13 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:14.059 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:13 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:14.059 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:13 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:14.059 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:13 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:14.059 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:13 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:14.059 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:13 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:14.059 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:13 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:55:14.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:55:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:55:14] "GET /metrics HTTP/1.1" 200 33893 "" "Prometheus/2.43.0" 2024-04-26T04:55:14.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:14 smithi156 ceph-mon[103290]: pgmap v355: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:14.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:14 smithi156 ceph-mon[103290]: pgmap v356: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:14.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:14 smithi156 ceph-mon[103290]: pgmap v357: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:14.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:14 smithi156 ceph-mon[98743]: pgmap v355: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:14.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:14 smithi156 ceph-mon[98743]: pgmap v356: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:14.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:14 smithi156 ceph-mon[98743]: pgmap v357: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:14.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:14 smithi171 ceph-mon[90566]: pgmap v355: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:14.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:14 smithi171 ceph-mon[90566]: pgmap v356: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:14.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:14 smithi171 ceph-mon[90566]: pgmap v357: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:15.672 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:55:15.672 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:55:15.994 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:15 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2276355482' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:15.994 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:15 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2276355482' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:15.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:15 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2276355482' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:16.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:16 smithi171 ceph-mon[90566]: pgmap v358: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:17.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:16 smithi156 ceph-mon[98743]: pgmap v358: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:17.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:16 smithi156 ceph-mon[103290]: pgmap v358: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:18.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:18 smithi171 ceph-mon[90566]: pgmap v359: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:19.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:18 smithi156 ceph-mon[103290]: pgmap v359: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:19.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:18 smithi156 ceph-mon[98743]: pgmap v359: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:19.429 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:55:19.694 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:20.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:20 smithi156 ceph-mon[98743]: pgmap v360: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:20.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:20.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:20 smithi156 ceph-mon[103290]: pgmap v360: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:20.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:20.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:20 smithi171 ceph-mon[90566]: pgmap v360: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:20.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:21.686 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:55:21.686 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:55:21.976 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:21 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1802567790' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:21.976 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:21 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1802567790' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:22.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:21 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1802567790' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:23.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:22 smithi156 ceph-mon[98743]: pgmap v361: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:23.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:22 smithi156 ceph-mon[103290]: pgmap v361: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:23.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:22 smithi171 ceph-mon[90566]: pgmap v361: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:25.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:24 smithi156 ceph-mon[98743]: pgmap v362: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:25.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:55:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:55:24] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.43.0" 2024-04-26T04:55:25.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:24 smithi156 ceph-mon[103290]: pgmap v362: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:25.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:24 smithi171 ceph-mon[90566]: pgmap v362: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:25.310 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:55:25.575 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:27.052 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:26 smithi171 ceph-mon[90566]: pgmap v363: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:27.083 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:26 smithi156 ceph-mon[98743]: pgmap v363: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:27.083 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:26 smithi156 ceph-mon[103290]: pgmap v363: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:27.570 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:55:27.570 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:55:27.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:27 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3628907195' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:27.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:27 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3628907195' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:28.245 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:27 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3628907195' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:29.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:28 smithi156 ceph-mon[98743]: pgmap v364: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:29.227 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:28 smithi156 ceph-mon[103290]: pgmap v364: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:29.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:28 smithi171 ceph-mon[90566]: pgmap v364: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:31.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:30 smithi156 ceph-mon[98743]: pgmap v365: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:31.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:30 smithi156 ceph-mon[103290]: pgmap v365: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:31.234 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:55:31.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:30 smithi171 ceph-mon[90566]: pgmap v365: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:31.501 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:32.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:32 smithi156 ceph-mon[98743]: pgmap v366: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:32.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:32 smithi156 ceph-mon[103290]: pgmap v366: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:33.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:32 smithi171 ceph-mon[90566]: pgmap v366: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:33.393 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:55:33.393 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:55:33.964 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:33 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3455120292' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:33.964 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:33 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3455120292' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:34.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:33 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3455120292' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:35.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:34 smithi156 ceph-mon[98743]: pgmap v367: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:35.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:55:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:55:34] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.43.0" 2024-04-26T04:55:35.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:34 smithi156 ceph-mon[103290]: pgmap v367: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:35.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:34 smithi171 ceph-mon[90566]: pgmap v367: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:36.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:36.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:36.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:35 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:36.989 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:55:37.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:36 smithi156 ceph-mon[98743]: pgmap v368: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:37.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:36 smithi156 ceph-mon[103290]: pgmap v368: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:37.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:36 smithi171 ceph-mon[90566]: pgmap v368: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:37.257 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:39.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:38 smithi156 ceph-mon[98743]: pgmap v369: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:39.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:38 smithi156 ceph-mon[103290]: pgmap v369: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:39.151 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:55:39.151 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:55:39.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:38 smithi171 ceph-mon[90566]: pgmap v369: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:40.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:39 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1200257166' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:40.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:39 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1200257166' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:40.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:39 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1200257166' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:41.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:40 smithi156 ceph-mon[98743]: pgmap v370: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:41.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:40 smithi156 ceph-mon[103290]: pgmap v370: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:41.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:40 smithi171 ceph-mon[90566]: pgmap v370: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:42.727 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:55:43.004 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:43.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:42 smithi156 ceph-mon[98743]: pgmap v371: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:43.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:42 smithi156 ceph-mon[103290]: pgmap v371: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:43.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:42 smithi171 ceph-mon[90566]: pgmap v371: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:44.889 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:44 smithi156 ceph-mon[98743]: pgmap v372: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:44.889 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:55:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:55:44] "GET /metrics HTTP/1.1" 200 33900 "" "Prometheus/2.43.0" 2024-04-26T04:55:44.889 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:44 smithi156 ceph-mon[103290]: pgmap v372: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:45.150 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:55:45.151 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:55:45.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:44 smithi171 ceph-mon[90566]: pgmap v372: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:46.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:45 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2367730398' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:46.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:45 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2367730398' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:46.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:45 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2367730398' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:47.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:46 smithi156 ceph-mon[98743]: pgmap v373: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:47.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:46 smithi156 ceph-mon[103290]: pgmap v373: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:47.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:46 smithi171 ceph-mon[90566]: pgmap v373: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:48.903 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:55:49.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:48 smithi156 ceph-mon[98743]: pgmap v374: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:49.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:48 smithi156 ceph-mon[103290]: pgmap v374: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:49.169 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:49.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:48 smithi171 ceph-mon[90566]: pgmap v374: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:51.194 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:55:51.194 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:55:51.195 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:50 smithi156 ceph-mon[98743]: pgmap v375: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:51.195 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:50 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:51.195 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:50 smithi156 ceph-mon[103290]: pgmap v375: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:51.195 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:50 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:51.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:50 smithi171 ceph-mon[90566]: pgmap v375: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:51.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:50 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:52.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:51 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1299650286' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:52.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:51 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1299650286' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:52.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:51 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1299650286' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:53.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:52 smithi171 ceph-mon[90566]: pgmap v376: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:53.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:52 smithi156 ceph-mon[103290]: pgmap v376: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:53.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:52 smithi156 ceph-mon[98743]: pgmap v376: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:54.817 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:55:54.961 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:55:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:55:54] "GET /metrics HTTP/1.1" 200 33896 "" "Prometheus/2.43.0" 2024-04-26T04:55:55.086 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:55.225 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:54 smithi156 ceph-mon[103290]: pgmap v377: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:55.225 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:54 smithi156 ceph-mon[98743]: pgmap v377: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:55.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:54 smithi171 ceph-mon[90566]: pgmap v377: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:57.013 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:55:57.013 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:55:57.014 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:56 smithi156 ceph-mon[98743]: pgmap v378: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:57.014 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:56 smithi156 ceph-mon[103290]: pgmap v378: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:57.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:56 smithi171 ceph-mon[90566]: pgmap v378: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:58.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:57 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1967352857' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:58.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:57 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1967352857' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:58.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:57 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1967352857' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:59.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:55:58 smithi171 ceph-mon[90566]: pgmap v379: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:59.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:55:58 smithi156 ceph-mon[98743]: pgmap v379: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:55:59.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:55:58 smithi156 ceph-mon[103290]: pgmap v379: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:00.832 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:56:01.098 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:01.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:00 smithi156 ceph-mon[98743]: pgmap v380: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:01.221 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:00 smithi156 ceph-mon[103290]: pgmap v380: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:01.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:00 smithi171 ceph-mon[90566]: pgmap v380: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:03.136 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:56:03.136 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:56:03.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:02 smithi171 ceph-mon[90566]: pgmap v381: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:03.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:02 smithi156 ceph-mon[98743]: pgmap v381: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:03.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:02 smithi156 ceph-mon[103290]: pgmap v381: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:04.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:03 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/4242849812' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:04.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:03 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/4242849812' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:04.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:03 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/4242849812' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:04.982 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:56:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:56:04] "GET /metrics HTTP/1.1" 200 33896 "" "Prometheus/2.43.0" 2024-04-26T04:56:05.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:04 smithi171 ceph-mon[90566]: pgmap v382: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:05.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:04 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:05.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:04 smithi156 ceph-mon[98743]: pgmap v382: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:05.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:04 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:05.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:04 smithi156 ceph-mon[103290]: pgmap v382: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:05.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:04 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:06.831 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:56:07.098 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:07.219 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:06 smithi156 ceph-mon[103290]: pgmap v383: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:07.219 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:06 smithi156 ceph-mon[98743]: pgmap v383: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:07.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:06 smithi171 ceph-mon[90566]: pgmap v383: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:09.066 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:56:09.066 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:56:09.079 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:08 smithi156 ceph-mon[98743]: pgmap v384: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:09.079 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:08 smithi156 ceph-mon[103290]: pgmap v384: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:09.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:08 smithi171 ceph-mon[90566]: pgmap v384: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:10.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:09 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2591318987' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:10.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:09 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2591318987' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:10.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:09 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2591318987' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:11.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:10 smithi171 ceph-mon[90566]: pgmap v385: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:11.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:10 smithi156 ceph-mon[98743]: pgmap v385: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:11.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:10 smithi156 ceph-mon[103290]: pgmap v385: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:12.713 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:56:12.981 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:13.249 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:12 smithi156 ceph-mon[98743]: pgmap v386: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:13.249 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:12 smithi156 ceph-mon[103290]: pgmap v386: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:13.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:12 smithi171 ceph-mon[90566]: pgmap v386: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:14.144 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:13 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:14.144 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:13 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:14.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:13 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:14.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:56:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:56:14] "GET /metrics HTTP/1.1" 200 33896 "" "Prometheus/2.43.0" 2024-04-26T04:56:14.982 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:56:14.983 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:56:15.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:15 smithi171 ceph-mon[90566]: pgmap v387: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:15.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:15 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1014834946' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:15.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:15 smithi156 ceph-mon[103290]: pgmap v387: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:15.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:15 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1014834946' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:15.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:15 smithi156 ceph-mon[98743]: pgmap v387: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:15.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:15 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1014834946' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:17.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:17 smithi156 ceph-mon[98743]: pgmap v388: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:17.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:17 smithi156 ceph-mon[103290]: pgmap v388: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:17.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:17 smithi171 ceph-mon[90566]: pgmap v388: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:18.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:18 smithi171 ceph-mon[90566]: pgmap v389: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:18.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:18 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:56:18.500 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:18 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:56:18.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:18 smithi156 ceph-mon[98743]: pgmap v389: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:18.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:18 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:56:18.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:18 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:56:18.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:18 smithi156 ceph-mon[103290]: pgmap v389: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:18.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:18 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:56:18.594 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:18 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:56:18.670 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:56:18.952 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:19.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:19 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:56:19.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:19 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:56:19.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:19 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:20.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:19 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:20.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:19 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:56:20.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:19 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:56:20.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:19 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:56:20.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:19 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:20.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:19 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:20.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:19 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:56:20.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:19 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:56:20.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:19 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:56:20.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:19 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:20.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:19 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:20.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:19 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:56:20.998 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:56:20.998 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:56:21.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:20 smithi156 ceph-mon[103290]: pgmap v390: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:21.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:20 smithi156 ceph-mon[103290]: pgmap v391: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:21.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:20 smithi156 ceph-mon[103290]: pgmap v392: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:21.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:21.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:20 smithi156 ceph-mon[98743]: pgmap v390: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:21.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:20 smithi156 ceph-mon[98743]: pgmap v391: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:21.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:20 smithi156 ceph-mon[98743]: pgmap v392: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:21.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:21.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:20 smithi171 ceph-mon[90566]: pgmap v390: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:21.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:20 smithi171 ceph-mon[90566]: pgmap v391: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:21.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:20 smithi171 ceph-mon[90566]: pgmap v392: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:21.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:22.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:21 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2703950776' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:22.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:21 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2703950776' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:22.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:21 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2703950776' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:23.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:22 smithi171 ceph-mon[90566]: pgmap v393: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:23.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:22 smithi156 ceph-mon[98743]: pgmap v393: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:23.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:22 smithi156 ceph-mon[103290]: pgmap v393: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:24.765 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:56:24.970 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:56:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:56:24] "GET /metrics HTTP/1.1" 200 33897 "" "Prometheus/2.43.0" 2024-04-26T04:56:25.023 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:25.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:24 smithi171 ceph-mon[90566]: pgmap v394: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:25.268 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:24 smithi156 ceph-mon[98743]: pgmap v394: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:25.268 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:24 smithi156 ceph-mon[103290]: pgmap v394: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:27.145 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:56:27.145 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:56:27.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:26 smithi171 ceph-mon[90566]: pgmap v395: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:27.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:26 smithi156 ceph-mon[98743]: pgmap v395: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:27.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:26 smithi156 ceph-mon[103290]: pgmap v395: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:28.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:27 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2795257234' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:28.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:27 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2795257234' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:28.239 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:27 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2795257234' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:29.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:28 smithi171 ceph-mon[90566]: pgmap v396: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:29.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:28 smithi156 ceph-mon[98743]: pgmap v396: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:29.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:28 smithi156 ceph-mon[103290]: pgmap v396: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:30.807 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:56:31.075 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:31.092 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:30 smithi156 ceph-mon[98743]: pgmap v397: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:31.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:30 smithi156 ceph-mon[103290]: pgmap v397: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:31.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:30 smithi171 ceph-mon[90566]: pgmap v397: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:33.103 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:56:33.103 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:56:33.116 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:32 smithi156 ceph-mon[98743]: pgmap v398: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:33.116 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:32 smithi156 ceph-mon[103290]: pgmap v398: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:33.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:32 smithi171 ceph-mon[90566]: pgmap v398: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:34.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:33 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3307874090' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:34.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:33 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3307874090' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:34.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:33 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3307874090' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:34.988 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:56:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:56:34] "GET /metrics HTTP/1.1" 200 33897 "" "Prometheus/2.43.0" 2024-04-26T04:56:35.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:34 smithi171 ceph-mon[90566]: pgmap v399: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:35.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:34 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:35.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:34 smithi156 ceph-mon[98743]: pgmap v399: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:35.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:34 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:35.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:34 smithi156 ceph-mon[103290]: pgmap v399: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:35.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:34 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:36.897 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:56:37.165 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:37.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:36 smithi171 ceph-mon[90566]: pgmap v400: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:37.287 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:36 smithi156 ceph-mon[98743]: pgmap v400: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:37.288 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:36 smithi156 ceph-mon[103290]: pgmap v400: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:39.221 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:56:39.221 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:56:39.222 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:38 smithi156 ceph-mon[98743]: pgmap v401: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:39.222 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:38 smithi156 ceph-mon[103290]: pgmap v401: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:39.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:38 smithi171 ceph-mon[90566]: pgmap v401: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:40.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:39 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/3023131992' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:40.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:39 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/3023131992' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:40.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:39 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/3023131992' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:41.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:40 smithi156 ceph-mon[98743]: pgmap v402: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:41.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:40 smithi156 ceph-mon[103290]: pgmap v402: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:41.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:40 smithi171 ceph-mon[90566]: pgmap v402: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:42.859 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:56:43.124 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:43.145 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:43 smithi156 ceph-mon[98743]: pgmap v403: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:43.146 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:43 smithi156 ceph-mon[103290]: pgmap v403: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:43.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:43 smithi171 ceph-mon[90566]: pgmap v403: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:45.004 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:56:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:56:44] "GET /metrics HTTP/1.1" 200 33897 "" "Prometheus/2.43.0" 2024-04-26T04:56:45.297 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:56:45.297 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:56:45.297 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:45 smithi156 ceph-mon[98743]: pgmap v404: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:45.297 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:45 smithi156 ceph-mon[103290]: pgmap v404: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:45.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:45 smithi171 ceph-mon[90566]: pgmap v404: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:46.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:46 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/854154232' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:46.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:46 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/854154232' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:46.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:46 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/854154232' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:47.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:47 smithi156 ceph-mon[98743]: pgmap v405: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:47.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:47 smithi156 ceph-mon[103290]: pgmap v405: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:47.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:47 smithi171 ceph-mon[90566]: pgmap v405: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:48.900 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:56:49.171 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:49.293 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:49 smithi156 ceph-mon[98743]: pgmap v406: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:49.294 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:49 smithi156 ceph-mon[103290]: pgmap v406: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:49.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:49 smithi171 ceph-mon[90566]: pgmap v406: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:50.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:50 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:50.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:50 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:50.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:50 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:51.246 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:56:51.247 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:56:51.259 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:51 smithi156 ceph-mon[103290]: pgmap v407: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:51.259 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:51 smithi156 ceph-mon[98743]: pgmap v407: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:51.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:51 smithi171 ceph-mon[90566]: pgmap v407: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:52.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:52 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/1523104784' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:52.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:52 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/1523104784' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:52.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:52 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/1523104784' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:53.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:53 smithi156 ceph-mon[98743]: pgmap v408: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:53.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:53 smithi156 ceph-mon[103290]: pgmap v408: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:53.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:53 smithi171 ceph-mon[90566]: pgmap v408: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:54.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:54 smithi156 ceph-mon[98743]: pgmap v409: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:54.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:54 smithi156 ceph-mon[103290]: pgmap v409: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:54.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:54 smithi171 ceph-mon[90566]: pgmap v409: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:54.989 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:56:55.016 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:56:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:56:54] "GET /metrics HTTP/1.1" 200 33893 "" "Prometheus/2.43.0" 2024-04-26T04:56:55.255 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:56.691 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:56 smithi156 ceph-mon[98743]: pgmap v410: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:56.691 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:56 smithi156 ceph-mon[103290]: pgmap v410: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:56.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:56 smithi171 ceph-mon[90566]: pgmap v410: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:57.291 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:56:57.291 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:56:57.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:57 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2063308259' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:57.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:57 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2063308259' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:57.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:57 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2063308259' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:58.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:56:58 smithi171 ceph-mon[90566]: pgmap v411: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:58.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:56:58 smithi156 ceph-mon[103290]: pgmap v411: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:56:58.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:56:58 smithi156 ceph-mon[98743]: pgmap v411: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:00.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:00 smithi171 ceph-mon[90566]: pgmap v412: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:00.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:00 smithi156 ceph-mon[98743]: pgmap v412: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:00.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:00 smithi156 ceph-mon[103290]: pgmap v412: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:00.978 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:57:01.244 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:57:02.576 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:02 smithi156 ceph-mon[98743]: pgmap v413: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:02.576 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:02 smithi156 ceph-mon[103290]: pgmap v413: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:02.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:02 smithi171 ceph-mon[90566]: pgmap v413: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:03.079 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:57:03.079 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:57:03.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:03 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2512484217' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:03.806 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:03 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2512484217' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:03.806 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:03 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2512484217' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:04.704 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:04 smithi156 ceph-mon[103290]: pgmap v414: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:04.705 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:04 smithi156 ceph-mon[98743]: pgmap v414: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:04.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:04 smithi171 ceph-mon[90566]: pgmap v414: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:05.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:57:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:57:04] "GET /metrics HTTP/1.1" 200 33893 "" "Prometheus/2.43.0" 2024-04-26T04:57:05.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:05 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:05.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:05 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:05.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:05 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:06.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:06 smithi171 ceph-mon[90566]: pgmap v415: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:06.832 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:57:06.842 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:06 smithi156 ceph-mon[103290]: pgmap v415: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:06.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:06 smithi156 ceph-mon[98743]: pgmap v415: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:07.095 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:57:08.658 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:08 smithi156 ceph-mon[103290]: pgmap v416: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:08.658 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:08 smithi156 ceph-mon[98743]: pgmap v416: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:08.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:08 smithi171 ceph-mon[90566]: pgmap v416: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:09.137 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:57:09.137 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:57:09.728 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:09 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2932744138' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:09.729 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:09 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2932744138' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:09.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:09 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2932744138' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:10.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:10 smithi171 ceph-mon[90566]: pgmap v417: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:10.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:10 smithi156 ceph-mon[98743]: pgmap v417: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:10.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:10 smithi156 ceph-mon[103290]: pgmap v417: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:12.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:12 smithi171 ceph-mon[90566]: pgmap v418: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:12.754 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:57:12.781 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:12 smithi156 ceph-mon[98743]: pgmap v418: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:12.782 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:12 smithi156 ceph-mon[103290]: pgmap v418: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:13.022 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:57:14.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:14 smithi156 ceph-mon[98743]: pgmap v419: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:14.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:14 smithi156 ceph-mon[103290]: pgmap v419: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:14.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:14 smithi171 ceph-mon[90566]: pgmap v419: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:15.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:57:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:57:14] "GET /metrics HTTP/1.1" 200 33894 "" "Prometheus/2.43.0" 2024-04-26T04:57:15.119 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:57:15.119 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:57:15.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:15 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/4126935108' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:15.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:15 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/4126935108' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:15.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:15 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/4126935108' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:16.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:16 smithi171 ceph-mon[90566]: pgmap v420: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:16.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:16 smithi156 ceph-mon[98743]: pgmap v420: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:16.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:16 smithi156 ceph-mon[103290]: pgmap v420: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:18.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:18 smithi171 ceph-mon[90566]: pgmap v421: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:18.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:18 smithi156 ceph-mon[103290]: pgmap v421: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:18.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:18 smithi156 ceph-mon[98743]: pgmap v421: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:18.896 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:57:19.162 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:57:20.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:20 smithi171 ceph-mon[90566]: pgmap v422: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:20.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:20.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:20 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:20.776 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:20 smithi156 ceph-mon[98743]: pgmap v422: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:20.776 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:20.776 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:20 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:20.776 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:20 smithi156 ceph-mon[103290]: pgmap v422: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:20.776 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:20.776 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:20 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:21.271 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:57:21.271 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:57:21.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:21 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/375195551' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:21.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:21 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/375195551' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:21.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:21 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/375195551' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:22.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:22 smithi171 ceph-mon[90566]: pgmap v423: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:22.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:22 smithi156 ceph-mon[98743]: pgmap v423: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:22.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:22 smithi156 ceph-mon[103290]: pgmap v423: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:24.704 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:24 smithi156 ceph-mon[98743]: pgmap v424: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:24.704 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:24 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:57:24.705 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:24 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:57:24.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:24 smithi156 ceph-mon[103290]: pgmap v424: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:24.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:24 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:57:24.705 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:24 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:57:24.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:24 smithi171 ceph-mon[90566]: pgmap v424: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:24.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:24 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:57:24.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:24 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:57:25.037 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:57:25.058 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:57:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:57:24] "GET /metrics HTTP/1.1" 200 33894 "" "Prometheus/2.43.0" 2024-04-26T04:57:25.305 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:57:26.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:25 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:57:26.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:25 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:57:26.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:25 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:26.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:25 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:26.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:25 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:57:26.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:25 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:57:26.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:25 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:57:26.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:25 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:26.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:25 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:26.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:25 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:57:26.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:25 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:57:26.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:25 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:57:26.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:25 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:26.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:25 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:26.250 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:25 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' 2024-04-26T04:57:27.151 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:26 smithi171 ceph-mon[90566]: pgmap v425: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:27.152 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:26 smithi171 ceph-mon[90566]: pgmap v426: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:27.152 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:26 smithi171 ceph-mon[90566]: pgmap v427: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:27.167 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:26 smithi156 ceph-mon[98743]: pgmap v425: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:27.168 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:26 smithi156 ceph-mon[98743]: pgmap v426: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:27.168 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:26 smithi156 ceph-mon[98743]: pgmap v427: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:27.168 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:26 smithi156 ceph-mon[103290]: pgmap v425: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:27.169 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:26 smithi156 ceph-mon[103290]: pgmap v426: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:27.169 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:26 smithi156 ceph-mon[103290]: pgmap v427: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:27.400 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:57:27.401 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:57:28.029 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:27 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2194269929' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:28.030 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:27 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2194269929' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:28.174 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:27 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2194269929' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:29.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:28 smithi171 ceph-mon[90566]: pgmap v428: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:29.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:28 smithi156 ceph-mon[98743]: pgmap v428: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:29.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:28 smithi156 ceph-mon[103290]: pgmap v428: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:31.057 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:57:31.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:30 smithi171 ceph-mon[90566]: pgmap v429: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:31.322 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:57:31.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:30 smithi156 ceph-mon[98743]: pgmap v429: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:31.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:30 smithi156 ceph-mon[103290]: pgmap v429: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:33.140 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:32 smithi156 ceph-mon[98743]: pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:33.140 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:32 smithi156 ceph-mon[103290]: pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:33.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:32 smithi171 ceph-mon[90566]: pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:33.382 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:57:33.382 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:57:34.077 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:33 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2703275125' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:34.078 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:33 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2703275125' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:34.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:33 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2703275125' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:35.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:34 smithi156 ceph-mon[98743]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:35.093 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:57:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:57:34] "GET /metrics HTTP/1.1" 200 33894 "" "Prometheus/2.43.0" 2024-04-26T04:57:35.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:34 smithi156 ceph-mon[103290]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:35.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:34 smithi171 ceph-mon[90566]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:36.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:35 smithi171 ceph-mon[90566]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:36.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:35 smithi156 ceph-mon[103290]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:36.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:35 smithi156 ceph-mon[98743]: from='mgr.25250 172.21.15.156:0/589031355' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:37.108 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:57:37.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:36 smithi171 ceph-mon[90566]: pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:37.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:36 smithi156 ceph-mon[98743]: pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:37.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:36 smithi156 ceph-mon[103290]: pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:37.372 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:57:39.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:38 smithi156 ceph-mon[98743]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:39.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:38 smithi156 ceph-mon[103290]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:39.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:38 smithi171 ceph-mon[90566]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:39.336 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:57:39.336 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:57:40.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:39 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/2182120486' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:40.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:39 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/2182120486' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:40.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:39 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/2182120486' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:41.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:40 smithi171 ceph-mon[90566]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:41.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:40 smithi156 ceph-mon[103290]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:41.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:40 smithi156 ceph-mon[98743]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:43.043 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 110027f2-0386-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:57:43.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:42 smithi171 ceph-mon[90566]: pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:43.321 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:57:43.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:42 smithi156 ceph-mon[103290]: pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:43.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:42 smithi156 ceph-mon[98743]: pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:44.840 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:57:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: ::ffff:172.21.15.171 - - [26/Apr/2024:04:57:44] "GET /metrics HTTP/1.1" 200 33896 "" "Prometheus/2.43.0" 2024-04-26T04:57:45.093 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:44 smithi156 ceph-mon[103290]: pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:45.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:44 smithi156 ceph-mon[98743]: pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:45.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:44 smithi171 ceph-mon[90566]: pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-26T04:57:45.335 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-04-26T04:57:45.335 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T04:57:46.066 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-26T04:57:46.304 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=7dfbfe6bc7c74b668350e5c08a4a6543 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-26T04:57:46.306 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-26T04:57:46.316 INFO:tasks.cephadm:Teardown begin 2024-04-26T04:57:46.316 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:57:46.320 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:45 smithi171 ceph-mon[90566]: from='client.? 172.21.15.156:0/4194197611' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:46.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:45 smithi156 ceph-mon[98743]: from='client.? 172.21.15.156:0/4194197611' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:46.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:45 smithi156 ceph-mon[103290]: from='client.? 172.21.15.156:0/4194197611' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:57:46.347 DEBUG:teuthology.orchestra.run.smithi171:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:57:46.381 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-26T04:57:46.381 DEBUG:teuthology.orchestra.run.smithi156:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-26T04:57:46.403 DEBUG:teuthology.orchestra.run.smithi171:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-26T04:57:46.438 INFO:tasks.cephadm:Stopping all daemons... 2024-04-26T04:57:46.438 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-26T04:57:46.438 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.a 2024-04-26T04:57:46.765 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:46 smithi156 systemd[1]: Stopping Ceph mon.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:57:47.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:46 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a[98720]: 2024-04-26T04:57:46.764+0000 7fc841db5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:57:47.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:46 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a[98720]: 2024-04-26T04:57:46.764+0000 7fc841db5700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-04-26T04:57:47.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:46 smithi156 podman[153137]: 2024-04-26 04:57:46.784950084 +0000 UTC m=+0.161385422 container died b94e4b5539a16e33be567f6e470d89308a5a962112f4bb5960609def847706ba (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-26T04:57:47.094 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:57:46 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:57:46] ENGINE Bus STOPPING 2024-04-26T04:57:47.094 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:57:46 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:57:46] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:57:47.094 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:57:46 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:57:46] ENGINE Bus STOPPED 2024-04-26T04:57:47.094 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:57:46 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:57:46] ENGINE Bus STARTING 2024-04-26T04:57:47.094 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:57:46 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:57:46] ENGINE Serving on http://:::9283 2024-04-26T04:57:47.094 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:57:46 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: [26/Apr/2024:04:57:46] ENGINE Bus STARTED 2024-04-26T04:57:47.388 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:47 smithi156 podman[153137]: 2024-04-26 04:57:47.186832693 +0000 UTC m=+0.563268029 container cleanup b94e4b5539a16e33be567f6e470d89308a5a962112f4bb5960609def847706ba (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424) 2024-04-26T04:57:47.388 INFO:journalctl@ceph.mon.a.smithi156.stdout:Apr 26 04:57:47 smithi156 bash[153137]: ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-a 2024-04-26T04:57:47.520 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.a.service' 2024-04-26T04:57:47.565 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T04:57:47.565 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-26T04:57:47.565 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-26T04:57:47.566 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.c 2024-04-26T04:57:47.843 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:47 smithi156 systemd[1]: Stopping Ceph mon.c for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:57:48.135 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:47 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c[103266]: 2024-04-26T04:57:47.867+0000 7f015a00a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:57:48.135 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:47 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c[103266]: 2024-04-26T04:57:47.867+0000 7f015a00a700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-04-26T04:57:48.459 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:48 smithi156 podman[153279]: 2024-04-26 04:57:48.134555687 +0000 UTC m=+0.378928862 container died 8cac6c96e0db9048064ab747ea8d4ffcc421a7f45968df33dc89a98301a05e07 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240424) 2024-04-26T04:57:48.792 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:48 smithi156 podman[153279]: 2024-04-26 04:57:48.558112523 +0000 UTC m=+0.802485688 container cleanup 8cac6c96e0db9048064ab747ea8d4ffcc421a7f45968df33dc89a98301a05e07 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-26T04:57:48.793 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:48 smithi156 bash[153279]: ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c 2024-04-26T04:57:49.031 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.c.service' 2024-04-26T04:57:49.052 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:48 smithi156 podman[153298]: 2024-04-26 04:57:48.79273876 +0000 UTC m=+0.657938405 container remove 8cac6c96e0db9048064ab747ea8d4ffcc421a7f45968df33dc89a98301a05e07 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-c, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_CLEAN=True, ceph=True) 2024-04-26T04:57:49.052 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:49 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.c.service: Deactivated successfully. 2024-04-26T04:57:49.052 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:49 smithi156 systemd[1]: Stopped Ceph mon.c for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:57:49.052 INFO:journalctl@ceph.mon.c.smithi156.stdout:Apr 26 04:57:49 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.c.service: Consumed 5.825s CPU time. 2024-04-26T04:57:49.065 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T04:57:49.065 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-26T04:57:49.065 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-26T04:57:49.065 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.b 2024-04-26T04:57:49.374 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:49 smithi171 systemd[1]: Stopping Ceph mon.b for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:57:49.682 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-b[90543]: 2024-04-26T04:57:49.372+0000 7fb08beba700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:57:49.682 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-b[90543]: 2024-04-26T04:57:49.372+0000 7fb08beba700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-04-26T04:57:49.682 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:49 smithi171 podman[105265]: 2024-04-26 04:57:49.384239169 +0000 UTC m=+0.126330975 container died 0829e2e150f0f5afd77de2d85eb01e6b1a0a409df8283ffa2464b9d1fc04d867 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-b, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-26T04:57:49.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:49 smithi171 podman[105265]: 2024-04-26 04:57:49.854700725 +0000 UTC m=+0.596792526 container cleanup 0829e2e150f0f5afd77de2d85eb01e6b1a0a409df8283ffa2464b9d1fc04d867 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS) 2024-04-26T04:57:49.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Apr 26 04:57:49 smithi171 bash[105265]: ceph-110027f2-0386-11ef-bc93-c7b262605968-mon-b 2024-04-26T04:57:50.254 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@mon.b.service' 2024-04-26T04:57:50.297 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T04:57:50.297 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-26T04:57:50.297 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-26T04:57:50.297 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.y 2024-04-26T04:57:50.591 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:57:50 smithi156 systemd[1]: Stopping Ceph mgr.y for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:57:50.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:57:50 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:57:50.589+0000 7f7453994700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:57:50.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:57:50 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y[89821]: 2024-04-26T04:57:50.589+0000 7f7453994700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-26T04:57:50.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:57:50 smithi156 podman[153409]: 2024-04-26 04:57:50.62217586 +0000 UTC m=+0.141219918 container died 715bee4dd354b9b2a0844de7cd979c1044b2c29962121334105eb8f66f5a8c2b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:57:51.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:57:51 smithi156 podman[153409]: 2024-04-26 04:57:51.096577391 +0000 UTC m=+0.615621482 container cleanup 715bee4dd354b9b2a0844de7cd979c1044b2c29962121334105eb8f66f5a8c2b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-26T04:57:51.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Apr 26 04:57:51 smithi156 bash[153409]: ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-y 2024-04-26T04:57:51.675 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.y.service' 2024-04-26T04:57:51.708 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T04:57:51.708 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-26T04:57:51.709 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-26T04:57:51.709 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.x 2024-04-26T04:57:51.999 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:57:51 smithi171 systemd[1]: Stopping Ceph mgr.x for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:57:52.344 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:57:52 smithi171 podman[105397]: 2024-04-26 04:57:52.060066391 +0000 UTC m=+0.156821967 container died 9ec4d348d704a8af1fb289e626d003b972b022f50119a3d4d7a6d9844c6f4001 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-26T04:57:52.702 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:57:52 smithi171 podman[105397]: 2024-04-26 04:57:52.434139461 +0000 UTC m=+0.530895045 container cleanup 9ec4d348d704a8af1fb289e626d003b972b022f50119a3d4d7a6d9844c6f4001 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-26T04:57:52.702 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Apr 26 04:57:52 smithi171 bash[105397]: ceph-110027f2-0386-11ef-bc93-c7b262605968-mgr-x 2024-04-26T04:57:52.857 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@mgr.x.service' 2024-04-26T04:57:52.889 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T04:57:52.890 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-26T04:57:52.890 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-26T04:57:52.890 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.0 2024-04-26T04:57:53.204 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:57:52 smithi156 systemd[1]: Stopping Ceph osd.0 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:57:53.593 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:57:53 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0[112905]: 2024-04-26T04:57:53.203+0000 7f858f550700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:57:53.593 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:57:53 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0[112905]: 2024-04-26T04:57:53.203+0000 7f858f550700 -1 osd.0 103 *** Got signal Terminated *** 2024-04-26T04:57:53.593 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:57:53 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0[112905]: 2024-04-26T04:57:53.203+0000 7f858f550700 -1 osd.0 103 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:57:58.904 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:57:58 smithi156 podman[153539]: 2024-04-26 04:57:58.627112244 +0000 UTC m=+5.547369277 container died 7d9e99b5875cced995406151ea450ff3594670a4a1bf388241eda514afa16d79 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424) 2024-04-26T04:57:59.343 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:57:59 smithi156 podman[153539]: 2024-04-26 04:57:59.084974462 +0000 UTC m=+6.005231495 container cleanup 7d9e99b5875cced995406151ea450ff3594670a4a1bf388241eda514afa16d79 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-26T04:57:59.343 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:57:59 smithi156 bash[153539]: ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0 2024-04-26T04:57:59.697 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:57:59 smithi156 podman[153560]: 2024-04-26 04:57:59.352916139 +0000 UTC m=+0.719066995 container remove 7d9e99b5875cced995406151ea450ff3594670a4a1bf388241eda514afa16d79 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2) 2024-04-26T04:58:00.049 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:57:59 smithi156 podman[153655]: 2024-04-26 04:57:59.599427038 +0000 UTC m=+0.017929281 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:58:00.049 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:57:59 smithi156 podman[153655]: 2024-04-26 04:57:59.712230605 +0000 UTC m=+0.130732846 container create becd1b6d3f6e1ed4fd5005e4d0e8fce4fce631517f534860b616cc3cc58a9958 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:58:00.049 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:57:59 smithi156 podman[153655]: 2024-04-26 04:57:59.940727012 +0000 UTC m=+0.359229262 container init becd1b6d3f6e1ed4fd5005e4d0e8fce4fce631517f534860b616cc3cc58a9958 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-deactivate, RELEASE=squid-b22e2eb, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-26T04:58:00.049 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:57:59 smithi156 podman[153655]: 2024-04-26 04:57:59.946759033 +0000 UTC m=+0.365261280 container start becd1b6d3f6e1ed4fd5005e4d0e8fce4fce631517f534860b616cc3cc58a9958 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424) 2024-04-26T04:58:00.050 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:58:00 smithi156 podman[153655]: 2024-04-26 04:58:00.048335417 +0000 UTC m=+0.466837674 container attach becd1b6d3f6e1ed4fd5005e4d0e8fce4fce631517f534860b616cc3cc58a9958 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-26T04:58:00.343 INFO:journalctl@ceph.osd.0.smithi156.stdout:Apr 26 04:58:00 smithi156 podman[153655]: 2024-04-26 04:58:00.245738876 +0000 UTC m=+0.664241121 container died becd1b6d3f6e1ed4fd5005e4d0e8fce4fce631517f534860b616cc3cc58a9958 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-26T04:58:01.016 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.0.service' 2024-04-26T04:58:01.054 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T04:58:01.054 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-26T04:58:01.054 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-26T04:58:01.055 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.1 2024-04-26T04:58:01.375 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:58:01 smithi156 systemd[1]: Stopping Ceph osd.1 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:58:01.843 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:58:01 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1[118057]: 2024-04-26T04:58:01.374+0000 7f506656b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:58:01.843 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:58:01 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1[118057]: 2024-04-26T04:58:01.374+0000 7f506656b700 -1 osd.1 103 *** Got signal Terminated *** 2024-04-26T04:58:01.843 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:58:01 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1[118057]: 2024-04-26T04:58:01.374+0000 7f506656b700 -1 osd.1 103 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:58:04.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:04.283Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi171:8443/api/prometheus_receiver\": dial tcp 172.21.15.171:8443: connect: connection refused" 2024-04-26T04:58:04.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:04 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:04.283Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused" 2024-04-26T04:58:07.048 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:58:06 smithi156 podman[153790]: 2024-04-26 04:58:06.758910911 +0000 UTC m=+5.490161311 container died 3e65e778dd5a67580d11dd2acdcbd163c6c59d275864a558de9af8f7fc9c7736 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-26T04:58:07.343 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:58:07 smithi156 podman[153790]: 2024-04-26 04:58:07.246615092 +0000 UTC m=+5.977865474 container cleanup 3e65e778dd5a67580d11dd2acdcbd163c6c59d275864a558de9af8f7fc9c7736 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:58:07.343 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:58:07 smithi156 bash[153790]: ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1 2024-04-26T04:58:07.803 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:58:07 smithi156 podman[153813]: 2024-04-26 04:58:07.48129279 +0000 UTC m=+0.724401572 container remove 3e65e778dd5a67580d11dd2acdcbd163c6c59d275864a558de9af8f7fc9c7736 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:58:08.093 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:58:07 smithi156 podman[153911]: 2024-04-26 04:58:07.706502005 +0000 UTC m=+0.018128393 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:58:08.093 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:58:07 smithi156 podman[153911]: 2024-04-26 04:58:07.80711724 +0000 UTC m=+0.118743631 container create a95f28520f3e06828f25badf14b44a13ce221996bed99981ccc9de5e2d6867a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_CLEAN=True) 2024-04-26T04:58:08.093 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:58:08 smithi156 podman[153911]: 2024-04-26 04:58:08.017959895 +0000 UTC m=+0.329586283 container init a95f28520f3e06828f25badf14b44a13ce221996bed99981ccc9de5e2d6867a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:58:08.093 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:58:08 smithi156 podman[153911]: 2024-04-26 04:58:08.023664932 +0000 UTC m=+0.335291328 container start a95f28520f3e06828f25badf14b44a13ce221996bed99981ccc9de5e2d6867a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424) 2024-04-26T04:58:08.593 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:58:08 smithi156 podman[153911]: 2024-04-26 04:58:08.101059995 +0000 UTC m=+0.412686403 container attach a95f28520f3e06828f25badf14b44a13ce221996bed99981ccc9de5e2d6867a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-26T04:58:08.593 INFO:journalctl@ceph.osd.1.smithi156.stdout:Apr 26 04:58:08 smithi156 podman[153911]: 2024-04-26 04:58:08.311216771 +0000 UTC m=+0.622843161 container died a95f28520f3e06828f25badf14b44a13ce221996bed99981ccc9de5e2d6867a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-26T04:58:08.961 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.1.service' 2024-04-26T04:58:08.999 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T04:58:08.999 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-26T04:58:08.999 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-26T04:58:08.999 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.2 2024-04-26T04:58:09.320 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:58:09 smithi156 systemd[1]: Stopping Ceph osd.2 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:58:09.593 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:58:09 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2[123614]: 2024-04-26T04:58:09.319+0000 7f07f9e1d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:58:09.593 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:58:09 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2[123614]: 2024-04-26T04:58:09.319+0000 7f07f9e1d700 -1 osd.2 103 *** Got signal Terminated *** 2024-04-26T04:58:09.593 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:58:09 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2[123614]: 2024-04-26T04:58:09.319+0000 7f07f9e1d700 -1 osd.2 103 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:58:14.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:14.282Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi171:8443/api/prometheus_receiver\": dial tcp 172.21.15.171:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused" 2024-04-26T04:58:14.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:14.283Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused" 2024-04-26T04:58:14.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:14 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:14.283Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi171:8443/api/prometheus_receiver\": dial tcp 172.21.15.171:8443: connect: connection refused" 2024-04-26T04:58:14.951 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:58:14 smithi156 podman[154040]: 2024-04-26 04:58:14.751863711 +0000 UTC m=+5.536186076 container died fd77f3d690838577cd22aa333ae2fadd385606cf4482b2f47a7c1bc536299ca3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:58:15.201 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:58:15 smithi156 podman[154040]: 2024-04-26 04:58:15.183136049 +0000 UTC m=+5.967458401 container cleanup fd77f3d690838577cd22aa333ae2fadd385606cf4482b2f47a7c1bc536299ca3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-26T04:58:15.201 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:58:15 smithi156 bash[154040]: ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2 2024-04-26T04:58:15.593 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:58:15 smithi156 podman[154062]: 2024-04-26 04:58:15.409617261 +0000 UTC m=+0.655600284 container remove fd77f3d690838577cd22aa333ae2fadd385606cf4482b2f47a7c1bc536299ca3 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-26T04:58:16.093 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:58:15 smithi156 podman[154170]: 2024-04-26 04:58:15.737381824 +0000 UTC m=+0.018104256 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:58:16.093 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:58:15 smithi156 podman[154170]: 2024-04-26 04:58:15.868795595 +0000 UTC m=+0.149518028 container create 31d52d3f414d1866f6c815a345a10ed13fc3454f22fe80d439c8a4ab1754e00d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:58:16.431 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:58:16 smithi156 podman[154170]: 2024-04-26 04:58:16.129726183 +0000 UTC m=+0.410448618 container init 31d52d3f414d1866f6c815a345a10ed13fc3454f22fe80d439c8a4ab1754e00d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux ) 2024-04-26T04:58:16.431 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:58:16 smithi156 podman[154170]: 2024-04-26 04:58:16.135566114 +0000 UTC m=+0.416288549 container start 31d52d3f414d1866f6c815a345a10ed13fc3454f22fe80d439c8a4ab1754e00d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-26T04:58:16.431 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:58:16 smithi156 podman[154170]: 2024-04-26 04:58:16.24651189 +0000 UTC m=+0.527234324 container attach 31d52d3f414d1866f6c815a345a10ed13fc3454f22fe80d439c8a4ab1754e00d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424) 2024-04-26T04:58:16.691 INFO:journalctl@ceph.osd.2.smithi156.stdout:Apr 26 04:58:16 smithi156 podman[154218]: 2024-04-26 04:58:16.45533562 +0000 UTC m=+0.018901145 container died 31d52d3f414d1866f6c815a345a10ed13fc3454f22fe80d439c8a4ab1754e00d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-26T04:58:16.969 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.2.service' 2024-04-26T04:58:17.007 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T04:58:17.007 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-26T04:58:17.007 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-26T04:58:17.007 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.3 2024-04-26T04:58:17.323 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:58:17 smithi156 systemd[1]: Stopping Ceph osd.3 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:58:17.593 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:58:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3[129212]: 2024-04-26T04:58:17.322+0000 7fa307a67700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:58:17.593 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:58:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3[129212]: 2024-04-26T04:58:17.322+0000 7fa307a67700 -1 osd.3 103 *** Got signal Terminated *** 2024-04-26T04:58:17.593 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:58:17 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3[129212]: 2024-04-26T04:58:17.322+0000 7fa307a67700 -1 osd.3 103 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:58:19.999 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:19 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:19.718+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:20.749 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:20 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:20.466+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:21.249 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:20 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:20.751+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:21.749 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:21.437+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:22.249 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:21.786+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:22.250 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:21 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:21.930+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:22.737 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:22 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:22.418+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:22.999 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:22 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:22.737+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:23.000 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:22 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:22.890+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:23.104 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:58:22 smithi156 podman[154303]: 2024-04-26 04:58:22.84752039 +0000 UTC m=+5.626673135 container died 698081f2c4f0d8fb6965b895cdfff1fd436f8a3bc8c95a1d81995f23fe364b6e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2) 2024-04-26T04:58:23.487 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:58:23 smithi156 podman[154303]: 2024-04-26 04:58:23.27719412 +0000 UTC m=+6.056346925 container cleanup 698081f2c4f0d8fb6965b895cdfff1fd436f8a3bc8c95a1d81995f23fe364b6e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:58:23.487 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:58:23 smithi156 bash[154303]: ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3 2024-04-26T04:58:23.749 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:23 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:23.388+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:23.842 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:58:23 smithi156 podman[154326]: 2024-04-26 04:58:23.48686916 +0000 UTC m=+0.633703450 container remove 698081f2c4f0d8fb6965b895cdfff1fd436f8a3bc8c95a1d81995f23fe364b6e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb) 2024-04-26T04:58:24.150 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:23 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:23.769+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:24.150 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:23 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:23.849+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:24.173 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:58:23 smithi156 podman[154425]: 2024-04-26 04:58:23.793175932 +0000 UTC m=+0.017783501 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:58:24.173 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:58:23 smithi156 podman[154425]: 2024-04-26 04:58:23.904463136 +0000 UTC m=+0.129070742 container create 7c6864022feaa11c92a05a67ea021b407a7ea09814c148c9243fa9d5b4952548 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb) 2024-04-26T04:58:24.173 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:58:24 smithi156 podman[154425]: 2024-04-26 04:58:24.090331444 +0000 UTC m=+0.314939019 container init 7c6864022feaa11c92a05a67ea021b407a7ea09814c148c9243fa9d5b4952548 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True) 2024-04-26T04:58:24.173 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:58:24 smithi156 podman[154425]: 2024-04-26 04:58:24.096065777 +0000 UTC m=+0.320673348 container start 7c6864022feaa11c92a05a67ea021b407a7ea09814c148c9243fa9d5b4952548 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-26T04:58:24.173 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:58:24 smithi156 podman[154425]: 2024-04-26 04:58:24.173130508 +0000 UTC m=+0.397738085 container attach 7c6864022feaa11c92a05a67ea021b407a7ea09814c148c9243fa9d5b4952548 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS) 2024-04-26T04:58:24.499 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:24 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[97653]: 2024-04-26T04:58:24.149+0000 7f7fa5ed2700 -1 osd.4 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.848167+0000 front 2024-04-26T04:57:57.848080+0000 (oldest deadline 2024-04-26T04:58:23.147899+0000) 2024-04-26T04:58:24.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:24 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:24.384+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:24.593 INFO:journalctl@ceph.osd.3.smithi156.stdout:Apr 26 04:58:24 smithi156 podman[154425]: 2024-04-26 04:58:24.386612863 +0000 UTC m=+0.611220435 container died 7c6864022feaa11c92a05a67ea021b407a7ea09814c148c9243fa9d5b4952548 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5) 2024-04-26T04:58:24.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:24.282Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi171:8443/api/prometheus_receiver\": dial tcp 172.21.15.171:8443: connect: connection refused" 2024-04-26T04:58:24.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:24.283Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused" 2024-04-26T04:58:24.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:24 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:24.283Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi171:8443/api/prometheus_receiver\": dial tcp 172.21.15.171:8443: connect: connection refused" 2024-04-26T04:58:24.968 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.3.service' 2024-04-26T04:58:25.007 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T04:58:25.007 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-26T04:58:25.008 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-26T04:58:25.008 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.4 2024-04-26T04:58:25.062 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:24 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:24.796+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:25.063 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:24 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:24.868+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:25.331 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:25 smithi171 systemd[1]: Stopping Ceph osd.4 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:58:25.331 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[97653]: 2024-04-26T04:58:25.127+0000 7f7fa5ed2700 -1 osd.4 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.848167+0000 front 2024-04-26T04:57:57.848080+0000 (oldest deadline 2024-04-26T04:58:23.147899+0000) 2024-04-26T04:58:25.749 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:25.361+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:25.750 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[97653]: 2024-04-26T04:58:25.330+0000 7f7fad8e6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:58:25.750 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[97653]: 2024-04-26T04:58:25.330+0000 7f7fad8e6700 -1 osd.4 103 *** Got signal Terminated *** 2024-04-26T04:58:25.750 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[97653]: 2024-04-26T04:58:25.330+0000 7f7fad8e6700 -1 osd.4 103 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:58:26.127 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:25.906+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:26.127 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:25 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:25.834+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:26.499 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:26 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[97653]: 2024-04-26T04:58:26.126+0000 7f7fa5ed2700 -1 osd.4 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.848167+0000 front 2024-04-26T04:57:57.848080+0000 (oldest deadline 2024-04-26T04:58:23.147899+0000) 2024-04-26T04:58:26.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:26 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:26.330+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:27.162 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:26 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:26.808+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:27.163 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:26 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:26.895+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:27.499 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:27 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[97653]: 2024-04-26T04:58:27.161+0000 7f7fa5ed2700 -1 osd.4 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.848167+0000 front 2024-04-26T04:57:57.848080+0000 (oldest deadline 2024-04-26T04:58:23.147899+0000) 2024-04-26T04:58:27.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:27 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:27.296+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:28.166 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:27 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:27.761+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:28.166 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:27 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:27.942+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:28.499 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:28 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[97653]: 2024-04-26T04:58:28.164+0000 7f7fa5ed2700 -1 osd.4 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.848167+0000 front 2024-04-26T04:57:57.848080+0000 (oldest deadline 2024-04-26T04:58:23.147899+0000) 2024-04-26T04:58:28.499 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:28 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:28.274+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:28.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:28 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:28.274+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:28.999 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:28 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:28.968+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:29.000 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:28 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:28.733+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:29.499 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:29 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:29.323+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:29.499 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:29 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:29.323+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:29.500 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:29 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[97653]: 2024-04-26T04:58:29.173+0000 7f7fa5ed2700 -1 osd.4 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.848167+0000 front 2024-04-26T04:57:57.848080+0000 (oldest deadline 2024-04-26T04:58:23.147899+0000) 2024-04-26T04:58:29.500 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:29 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[97653]: 2024-04-26T04:58:29.173+0000 7f7fa5ed2700 -1 osd.4 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:06.048902+0000 front 2024-04-26T04:58:06.048940+0000 (oldest deadline 2024-04-26T04:58:28.348724+0000) 2024-04-26T04:58:29.968 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:29 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:29.716+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:29.969 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:29 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:29.716+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:02.970888+0000 front 2024-04-26T04:58:02.970929+0000 (oldest deadline 2024-04-26T04:58:28.870805+0000) 2024-04-26T04:58:30.249 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:29 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:29.968+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:30.250 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:30 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[97653]: 2024-04-26T04:58:30.195+0000 7f7fa5ed2700 -1 osd.4 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.848167+0000 front 2024-04-26T04:57:57.848080+0000 (oldest deadline 2024-04-26T04:58:23.147899+0000) 2024-04-26T04:58:30.250 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:30 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4[97653]: 2024-04-26T04:58:30.195+0000 7f7fa5ed2700 -1 osd.4 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:06.048902+0000 front 2024-04-26T04:58:06.048940+0000 (oldest deadline 2024-04-26T04:58:28.348724+0000) 2024-04-26T04:58:30.674 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:30 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:30.312+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:30.674 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:30 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:30.312+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:30.999 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:30 smithi171 podman[105542]: 2024-04-26 04:58:30.786199172 +0000 UTC m=+5.575840316 container died 9b3702c8c233908df777ca1736d4e6b713e47bbf87dc2b74525544e5c593f151 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb) 2024-04-26T04:58:31.000 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:30 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:30.673+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:31.000 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:30 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:30.673+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:02.970888+0000 front 2024-04-26T04:58:02.970929+0000 (oldest deadline 2024-04-26T04:58:28.870805+0000) 2024-04-26T04:58:31.273 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:31 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:31.004+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:31.273 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:31 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:31.004+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:31.273 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:31 smithi171 podman[105542]: 2024-04-26 04:58:31.169028773 +0000 UTC m=+5.958669917 container cleanup 9b3702c8c233908df777ca1736d4e6b713e47bbf87dc2b74525544e5c593f151 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-26T04:58:31.273 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:31 smithi171 bash[105542]: ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4 2024-04-26T04:58:31.632 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:31 smithi171 podman[105565]: 2024-04-26 04:58:31.386935068 +0000 UTC m=+0.598368694 container remove 9b3702c8c233908df777ca1736d4e6b713e47bbf87dc2b74525544e5c593f151 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-26T04:58:31.632 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:31 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:31.272+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:31.632 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:31 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:31.272+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:31.949 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:31 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:31.631+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:31.950 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:31 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:31.631+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:02.970888+0000 front 2024-04-26T04:58:02.970929+0000 (oldest deadline 2024-04-26T04:58:28.870805+0000) 2024-04-26T04:58:31.950 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:31 smithi171 podman[105660]: 2024-04-26 04:58:31.671205068 +0000 UTC m=+0.096830082 container create f05a03221130003792549c21840a4267be6a79d0240850c7ae8ad6ce8406c5fb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-26T04:58:31.950 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:31 smithi171 podman[105660]: 2024-04-26 04:58:31.59262164 +0000 UTC m=+0.018246678 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:58:32.249 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:32.018+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:32.249 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:32.018+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:32.250 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:31 smithi171 podman[105660]: 2024-04-26 04:58:31.949314878 +0000 UTC m=+0.374939902 container init f05a03221130003792549c21840a4267be6a79d0240850c7ae8ad6ce8406c5fb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-26T04:58:32.250 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:31 smithi171 podman[105660]: 2024-04-26 04:58:31.955079949 +0000 UTC m=+0.380704963 container start f05a03221130003792549c21840a4267be6a79d0240850c7ae8ad6ce8406c5fb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-26T04:58:32.250 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:32 smithi171 podman[105660]: 2024-04-26 04:58:32.04063323 +0000 UTC m=+0.466258261 container attach f05a03221130003792549c21840a4267be6a79d0240850c7ae8ad6ce8406c5fb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-26T04:58:32.522 INFO:journalctl@ceph.osd.4.smithi171.stdout:Apr 26 04:58:32 smithi171 podman[105660]: 2024-04-26 04:58:32.253673264 +0000 UTC m=+0.679298276 container died f05a03221130003792549c21840a4267be6a79d0240850c7ae8ad6ce8406c5fb (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-26T04:58:32.523 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:32.297+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:32.523 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:32.297+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:32.777 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:32.591+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:32.777 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:32 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:32.591+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:02.970888+0000 front 2024-04-26T04:58:02.970929+0000 (oldest deadline 2024-04-26T04:58:28.870805+0000) 2024-04-26T04:58:32.796 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.4.service' 2024-04-26T04:58:32.834 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T04:58:32.834 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-26T04:58:32.835 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-26T04:58:32.835 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.5 2024-04-26T04:58:33.175 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:32 smithi171 systemd[1]: Stopping Ceph osd.5 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:58:33.176 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:33 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:33.002+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:33.176 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:33 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:33.002+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:33.485 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:33 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:33.257+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:33.485 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:33 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:33.257+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:33.486 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:33 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:33.174+0000 7f8b64a4a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:58:33.486 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:33 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:33.174+0000 7f8b64a4a700 -1 osd.5 103 *** Got signal Terminated *** 2024-04-26T04:58:33.486 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:33 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:33.174+0000 7f8b64a4a700 -1 osd.5 103 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:58:33.749 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:33 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:33.618+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:33.749 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:33 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:33.618+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:02.970888+0000 front 2024-04-26T04:58:02.970929+0000 (oldest deadline 2024-04-26T04:58:28.870805+0000) 2024-04-26T04:58:34.499 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:34.218+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:34.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:34.218+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:34.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:34.049+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:34.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:34.049+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:34.500 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:58:34.225Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.156:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.156:8765: connect: connection refused" 2024-04-26T04:58:34.501 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:58:34.231Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.156:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.156:8765: connect: connection refused" 2024-04-26T04:58:34.501 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:58:34.231Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.156:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.156:8765: connect: connection refused" 2024-04-26T04:58:34.501 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:58:34.231Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.156:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.156:8765: connect: connection refused" 2024-04-26T04:58:34.501 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:58:34.231Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.156:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.156:8765: connect: connection refused" 2024-04-26T04:58:34.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:34.283Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi171:8443/api/prometheus_receiver\": dial tcp 172.21.15.171:8443: connect: connection refused" 2024-04-26T04:58:34.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:34.284Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused" 2024-04-26T04:58:34.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:34 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:34.284Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi171:8443/api/prometheus_receiver\": dial tcp 172.21.15.171:8443: connect: connection refused" 2024-04-26T04:58:34.999 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:34.575+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:34.999 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:34 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:34.575+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:02.970888+0000 front 2024-04-26T04:58:02.970929+0000 (oldest deadline 2024-04-26T04:58:28.870805+0000) 2024-04-26T04:58:35.499 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:35 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:35.240+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:35.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:35 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:35.240+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:35.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:35 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:35.067+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:35.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:35 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:35.067+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:35.999 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:35 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:35.544+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:35.999 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:35 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:35.544+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:02.970888+0000 front 2024-04-26T04:58:02.970929+0000 (oldest deadline 2024-04-26T04:58:28.870805+0000) 2024-04-26T04:58:36.499 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:36 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:36.231+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:36.499 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:36 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:36.231+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:36.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:36 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:36.036+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:36.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:36 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:36.036+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:36.999 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:36 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:36.559+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:37.000 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:36 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:36.559+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:02.970888+0000 front 2024-04-26T04:58:02.970929+0000 (oldest deadline 2024-04-26T04:58:28.870805+0000) 2024-04-26T04:58:37.000 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:36 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:36.559+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:11.171633+0000 front 2024-04-26T04:58:11.171574+0000 (oldest deadline 2024-04-26T04:58:35.871408+0000) 2024-04-26T04:58:37.499 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:37 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:37.242+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:37.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:37 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:37.242+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:37.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:37 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:37.041+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:37.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:37 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:37.041+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:37.999 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:37 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:37.577+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:54.170153+0000 front 2024-04-26T04:57:54.170223+0000 (oldest deadline 2024-04-26T04:58:18.870023+0000) 2024-04-26T04:58:38.000 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:37 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:37.577+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:02.970888+0000 front 2024-04-26T04:58:02.970929+0000 (oldest deadline 2024-04-26T04:58:28.870805+0000) 2024-04-26T04:58:38.000 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:37 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5[42950]: 2024-04-26T04:58:37.577+0000 7f8b5e843700 -1 osd.5 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:11.171633+0000 front 2024-04-26T04:58:11.171574+0000 (oldest deadline 2024-04-26T04:58:35.871408+0000) 2024-04-26T04:58:38.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:38 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:38.271+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:38.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:38 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:38.271+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:38.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:38 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:38.060+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:38.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:38 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:38.060+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:38.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:38 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:38.060+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:12.688782+0000 front 2024-04-26T04:58:12.688865+0000 (oldest deadline 2024-04-26T04:58:37.889290+0000) 2024-04-26T04:58:38.999 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:38 smithi171 podman[105801]: 2024-04-26 04:58:38.721340656 +0000 UTC m=+5.658140993 container died 703235a355b5f9a1cd7bcfcfa5c621fce68ec89d6520a15ddb804f2f3d6efe9f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-26T04:58:39.365 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:39 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:39.022+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:39.365 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:39 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:39.022+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:39.365 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:39 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:39.022+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:12.688782+0000 front 2024-04-26T04:58:12.688865+0000 (oldest deadline 2024-04-26T04:58:37.889290+0000) 2024-04-26T04:58:39.366 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:39 smithi171 podman[105801]: 2024-04-26 04:58:39.105022336 +0000 UTC m=+6.041822718 container cleanup 703235a355b5f9a1cd7bcfcfa5c621fce68ec89d6520a15ddb804f2f3d6efe9f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c) 2024-04-26T04:58:39.366 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:39 smithi171 bash[105801]: ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5 2024-04-26T04:58:39.366 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:39 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:39.243+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:39.366 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:39 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:39.243+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:39.721 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:39 smithi171 podman[105824]: 2024-04-26 04:58:39.364378785 +0000 UTC m=+0.640916630 container remove 703235a355b5f9a1cd7bcfcfa5c621fce68ec89d6520a15ddb804f2f3d6efe9f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True) 2024-04-26T04:58:39.999 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:39 smithi171 podman[105921]: 2024-04-26 04:58:39.624201973 +0000 UTC m=+0.018207031 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T04:58:40.000 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:39 smithi171 podman[105921]: 2024-04-26 04:58:39.740287582 +0000 UTC m=+0.134292625 container create c7b7834a36c20554368f835806c10095c8e39315dc78185731e5b8e97aaf7b91 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-26T04:58:40.000 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:39 smithi171 podman[105921]: 2024-04-26 04:58:39.917741496 +0000 UTC m=+0.311746542 container init c7b7834a36c20554368f835806c10095c8e39315dc78185731e5b8e97aaf7b91 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240424) 2024-04-26T04:58:40.000 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:39 smithi171 podman[105921]: 2024-04-26 04:58:39.922941375 +0000 UTC m=+0.316946425 container start c7b7834a36c20554368f835806c10095c8e39315dc78185731e5b8e97aaf7b91 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, RELEASE=quincy-c61472c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:58:40.295 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:40 smithi171 podman[105921]: 2024-04-26 04:58:40.000715804 +0000 UTC m=+0.394720858 container attach c7b7834a36c20554368f835806c10095c8e39315dc78185731e5b8e97aaf7b91 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-26T04:58:40.295 INFO:journalctl@ceph.osd.5.smithi171.stdout:Apr 26 04:58:40 smithi171 podman[105921]: 2024-04-26 04:58:40.205175381 +0000 UTC m=+0.599180428 container died c7b7834a36c20554368f835806c10095c8e39315dc78185731e5b8e97aaf7b91 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-5-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-26T04:58:40.295 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:40.035+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:40.295 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:40.035+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:40.296 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:40.035+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:12.688782+0000 front 2024-04-26T04:58:12.688865+0000 (oldest deadline 2024-04-26T04:58:37.889290+0000) 2024-04-26T04:58:40.737 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:40.293+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:40.738 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:40.293+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:40.738 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:40 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:40.293+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:14.242326+0000 front 2024-04-26T04:58:14.242360+0000 (oldest deadline 2024-04-26T04:58:39.542176+0000) 2024-04-26T04:58:40.755 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.5.service' 2024-04-26T04:58:40.790 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T04:58:40.791 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-26T04:58:40.791 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-26T04:58:40.791 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.6 2024-04-26T04:58:40.999 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:40 smithi171 systemd[1]: Stopping Ceph osd.6 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:58:41.499 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:41 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:41.067+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:41.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:41 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:41.067+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:41.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:41 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:41.067+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:12.688782+0000 front 2024-04-26T04:58:12.688865+0000 (oldest deadline 2024-04-26T04:58:37.889290+0000) 2024-04-26T04:58:41.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:41 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:41.125+0000 7f0f92b4a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:58:41.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:41 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:41.125+0000 7f0f92b4a700 -1 osd.6 103 *** Got signal Terminated *** 2024-04-26T04:58:41.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:41 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:41.125+0000 7f0f92b4a700 -1 osd.6 103 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:58:41.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:41 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:41.275+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:41.501 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:41 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:41.275+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:41.501 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:41 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:41.275+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:14.242326+0000 front 2024-04-26T04:58:14.242360+0000 (oldest deadline 2024-04-26T04:58:39.542176+0000) 2024-04-26T04:58:42.499 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:42.264+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:42.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:42.264+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:42.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:42.264+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:14.242326+0000 front 2024-04-26T04:58:14.242360+0000 (oldest deadline 2024-04-26T04:58:39.542176+0000) 2024-04-26T04:58:42.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:42.034+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:42.501 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:42.034+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:42.501 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:42.034+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:12.688782+0000 front 2024-04-26T04:58:12.688865+0000 (oldest deadline 2024-04-26T04:58:37.889290+0000) 2024-04-26T04:58:43.240 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:42.991+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:43.241 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:42.991+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:43.241 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:42 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:42.991+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:12.688782+0000 front 2024-04-26T04:58:12.688865+0000 (oldest deadline 2024-04-26T04:58:37.889290+0000) 2024-04-26T04:58:43.499 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:43 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:43.239+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:43.499 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:43 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:43.239+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:43.499 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:43 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:43.239+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:14.242326+0000 front 2024-04-26T04:58:14.242360+0000 (oldest deadline 2024-04-26T04:58:39.542176+0000) 2024-04-26T04:58:44.499 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:44 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:44.257+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:44.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:44 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:44.257+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:44.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:44 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:44.257+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:14.242326+0000 front 2024-04-26T04:58:14.242360+0000 (oldest deadline 2024-04-26T04:58:39.542176+0000) 2024-04-26T04:58:44.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:44 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:44.013+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:44.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:44 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:44.013+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:44.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:44 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:44.013+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:12.688782+0000 front 2024-04-26T04:58:12.688865+0000 (oldest deadline 2024-04-26T04:58:37.889290+0000) 2024-04-26T04:58:44.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:44 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:44.013+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-04-26T04:58:18.990401+0000 front 2024-04-26T04:58:18.990420+0000 (oldest deadline 2024-04-26T04:58:43.090266+0000) 2024-04-26T04:58:44.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:44.283Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi171:8443/api/prometheus_receiver\": dial tcp 172.21.15.171:8443: connect: connection refused" 2024-04-26T04:58:44.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:44.284Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi171:8443/api/prometheus_receiver\": dial tcp 172.21.15.171:8443: connect: connection refused" 2024-04-26T04:58:44.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:44 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:44.284Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused" 2024-04-26T04:58:45.499 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:45.245+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:56.239908+0000 front 2024-04-26T04:57:56.239835+0000 (oldest deadline 2024-04-26T04:58:19.739622+0000) 2024-04-26T04:58:45.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:45.245+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.640797+0000 front 2024-04-26T04:58:05.640817+0000 (oldest deadline 2024-04-26T04:58:27.840938+0000) 2024-04-26T04:58:45.500 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6[48949]: 2024-04-26T04:58:45.245+0000 7f0f9094b700 -1 osd.6 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:14.242326+0000 front 2024-04-26T04:58:14.242360+0000 (oldest deadline 2024-04-26T04:58:39.542176+0000) 2024-04-26T04:58:45.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:45.015+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:45.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:45.015+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:45.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:45.015+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:12.688782+0000 front 2024-04-26T04:58:12.688865+0000 (oldest deadline 2024-04-26T04:58:37.889290+0000) 2024-04-26T04:58:45.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:45.015+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-04-26T04:58:18.990401+0000 front 2024-04-26T04:58:18.990420+0000 (oldest deadline 2024-04-26T04:58:43.090266+0000) 2024-04-26T04:58:46.249 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:45.991+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:46.249 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:45.991+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:46.250 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:45.991+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:12.688782+0000 front 2024-04-26T04:58:12.688865+0000 (oldest deadline 2024-04-26T04:58:37.889290+0000) 2024-04-26T04:58:46.250 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:45 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:45.991+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-04-26T04:58:18.990401+0000 front 2024-04-26T04:58:18.990420+0000 (oldest deadline 2024-04-26T04:58:43.090266+0000) 2024-04-26T04:58:46.900 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:46 smithi171 podman[106056]: 2024-04-26 04:58:46.643176862 +0000 UTC m=+5.626000100 container died 3dad877f8247134e0dacd3fd96a8d87b8dab2e15212f3b1ebdae7842db2446ce (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:58:47.249 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:47 smithi171 podman[106056]: 2024-04-26 04:58:47.006843509 +0000 UTC m=+5.989666756 container cleanup 3dad877f8247134e0dacd3fd96a8d87b8dab2e15212f3b1ebdae7842db2446ce (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-26T04:58:47.250 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:47 smithi171 bash[106056]: ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6 2024-04-26T04:58:47.250 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:46 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:46.963+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:47.250 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:46 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:46.963+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:47.250 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:46 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:46.963+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:12.688782+0000 front 2024-04-26T04:58:12.688865+0000 (oldest deadline 2024-04-26T04:58:37.889290+0000) 2024-04-26T04:58:47.250 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:46 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:46.963+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-04-26T04:58:18.990401+0000 front 2024-04-26T04:58:18.990420+0000 (oldest deadline 2024-04-26T04:58:43.090266+0000) 2024-04-26T04:58:47.617 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:47 smithi171 podman[106078]: 2024-04-26 04:58:47.266451854 +0000 UTC m=+0.617842396 container remove 3dad877f8247134e0dacd3fd96a8d87b8dab2e15212f3b1ebdae7842db2446ce (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-26T04:58:47.869 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:47 smithi171 podman[106180]: 2024-04-26 04:58:47.617137804 +0000 UTC m=+0.110095336 container create b9d9a88d41695e30a5f28b2239a0281d7dc8f9e3af5704f7674b76e535d35867 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:58:47.870 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:47 smithi171 podman[106180]: 2024-04-26 04:58:47.525601637 +0000 UTC m=+0.018559186 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T04:58:47.870 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:47 smithi171 podman[106180]: 2024-04-26 04:58:47.869531375 +0000 UTC m=+0.362488910 container init b9d9a88d41695e30a5f28b2239a0281d7dc8f9e3af5704f7674b76e535d35867 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c) 2024-04-26T04:58:48.152 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:47 smithi171 podman[106180]: 2024-04-26 04:58:47.874962734 +0000 UTC m=+0.367920270 container start b9d9a88d41695e30a5f28b2239a0281d7dc8f9e3af5704f7674b76e535d35867 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6-deactivate, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True) 2024-04-26T04:58:48.152 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:47 smithi171 podman[106180]: 2024-04-26 04:58:47.944461028 +0000 UTC m=+0.437418561 container attach b9d9a88d41695e30a5f28b2239a0281d7dc8f9e3af5704f7674b76e535d35867 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c) 2024-04-26T04:58:48.152 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:47 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:47.996+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:48.152 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:47 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:47.996+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:48.153 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:47 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:47.996+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:12.688782+0000 front 2024-04-26T04:58:12.688865+0000 (oldest deadline 2024-04-26T04:58:37.889290+0000) 2024-04-26T04:58:48.153 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:47 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:47.996+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-04-26T04:58:18.990401+0000 front 2024-04-26T04:58:18.990420+0000 (oldest deadline 2024-04-26T04:58:43.090266+0000) 2024-04-26T04:58:48.431 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:48 smithi171 podman[106180]: 2024-04-26 04:58:48.152203577 +0000 UTC m=+0.645161108 container died b9d9a88d41695e30a5f28b2239a0281d7dc8f9e3af5704f7674b76e535d35867 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-26T04:58:48.684 INFO:journalctl@ceph.osd.6.smithi171.stdout:Apr 26 04:58:48 smithi171 podman[106226]: 2024-04-26 04:58:48.681146746 +0000 UTC m=+0.523606749 container remove b9d9a88d41695e30a5f28b2239a0281d7dc8f9e3af5704f7674b76e535d35867 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-26T04:58:48.701 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.6.service' 2024-04-26T04:58:48.738 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T04:58:48.738 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-26T04:58:48.738 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-26T04:58:48.738 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.7 2024-04-26T04:58:48.999 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:48 smithi171 systemd[1]: Stopping Ceph osd.7 for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:58:49.499 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:49.045+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:49.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:49.045+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:49.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:49.045+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:12.688782+0000 front 2024-04-26T04:58:12.688865+0000 (oldest deadline 2024-04-26T04:58:37.889290+0000) 2024-04-26T04:58:49.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:49.045+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-04-26T04:58:18.990401+0000 front 2024-04-26T04:58:18.990420+0000 (oldest deadline 2024-04-26T04:58:43.090266+0000) 2024-04-26T04:58:49.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:49.060+0000 7fd05c3c2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:58:49.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:49.060+0000 7fd05c3c2700 -1 osd.7 103 *** Got signal Terminated *** 2024-04-26T04:58:49.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:49 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:49.060+0000 7fd05c3c2700 -1 osd.7 103 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:58:50.499 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:50 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:50.017+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:50.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:50 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:50.017+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:50.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:50 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:50.017+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:12.688782+0000 front 2024-04-26T04:58:12.688865+0000 (oldest deadline 2024-04-26T04:58:37.889290+0000) 2024-04-26T04:58:50.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:50 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:50.017+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-04-26T04:58:18.990401+0000 front 2024-04-26T04:58:18.990420+0000 (oldest deadline 2024-04-26T04:58:43.090266+0000) 2024-04-26T04:58:51.499 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:51 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:51.006+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:51.499 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:51 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:51.006+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:51.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:51 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:51.006+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:12.688782+0000 front 2024-04-26T04:58:12.688865+0000 (oldest deadline 2024-04-26T04:58:37.889290+0000) 2024-04-26T04:58:51.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:51 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:51.006+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-04-26T04:58:18.990401+0000 front 2024-04-26T04:58:18.990420+0000 (oldest deadline 2024-04-26T04:58:43.090266+0000) 2024-04-26T04:58:52.249 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:51 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:51.974+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:52.250 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:51 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:51.974+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:52.250 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:51 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:51.974+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:12.688782+0000 front 2024-04-26T04:58:12.688865+0000 (oldest deadline 2024-04-26T04:58:37.889290+0000) 2024-04-26T04:58:52.250 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:51 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:51.974+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-04-26T04:58:18.990401+0000 front 2024-04-26T04:58:18.990420+0000 (oldest deadline 2024-04-26T04:58:43.090266+0000) 2024-04-26T04:58:53.499 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:53 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:53.011+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:53.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:53 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:53.011+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:53.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:53 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:53.011+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:12.688782+0000 front 2024-04-26T04:58:12.688865+0000 (oldest deadline 2024-04-26T04:58:37.889290+0000) 2024-04-26T04:58:53.500 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:53 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:53.011+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-04-26T04:58:18.990401+0000 front 2024-04-26T04:58:18.990420+0000 (oldest deadline 2024-04-26T04:58:43.090266+0000) 2024-04-26T04:58:54.390 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:54 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:54.044+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-04-26T04:57:57.785906+0000 front 2024-04-26T04:57:57.785883+0000 (oldest deadline 2024-04-26T04:58:21.285625+0000) 2024-04-26T04:58:54.390 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:54 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:54.044+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-04-26T04:58:05.287166+0000 front 2024-04-26T04:58:05.287062+0000 (oldest deadline 2024-04-26T04:58:30.487332+0000) 2024-04-26T04:58:54.390 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:54 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:54.044+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-04-26T04:58:12.688782+0000 front 2024-04-26T04:58:12.688865+0000 (oldest deadline 2024-04-26T04:58:37.889290+0000) 2024-04-26T04:58:54.390 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:54 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:54.044+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-04-26T04:58:18.990401+0000 front 2024-04-26T04:58:18.990420+0000 (oldest deadline 2024-04-26T04:58:43.090266+0000) 2024-04-26T04:58:54.391 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:54 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7[55110]: 2024-04-26T04:58:54.044+0000 7fd05a1c3700 -1 osd.7 103 heartbeat_check: no reply from 172.21.15.171:6804 osd.4 since back 2024-04-26T04:58:28.891782+0000 front 2024-04-26T04:58:28.891805+0000 (oldest deadline 2024-04-26T04:58:53.591648+0000) 2024-04-26T04:58:54.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:54.285Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi171:8443/api/prometheus_receiver\": dial tcp 172.21.15.171:8443: connect: connection refused" 2024-04-26T04:58:54.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:54.286Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused" 2024-04-26T04:58:54.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:54 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:54.286Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi171:8443/api/prometheus_receiver\": dial tcp 172.21.15.171:8443: connect: connection refused" 2024-04-26T04:58:54.668 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:54 smithi171 podman[106310]: 2024-04-26 04:58:54.389072657 +0000 UTC m=+5.438623932 container died e275b2da9b18df8756a6fe446c2e0926731d9b4f3ef3b8b84b08fa488181bcb5 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424) 2024-04-26T04:58:54.999 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:54 smithi171 podman[106310]: 2024-04-26 04:58:54.75009434 +0000 UTC m=+5.799645594 container cleanup e275b2da9b18df8756a6fe446c2e0926731d9b4f3ef3b8b84b08fa488181bcb5 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-26T04:58:54.999 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:54 smithi171 bash[106310]: ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7 2024-04-26T04:58:55.374 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:55 smithi171 podman[106331]: 2024-04-26 04:58:55.026159563 +0000 UTC m=+0.639794206 container remove e275b2da9b18df8756a6fe446c2e0926731d9b4f3ef3b8b84b08fa488181bcb5 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, ceph=True) 2024-04-26T04:58:55.696 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:55 smithi171 podman[106429]: 2024-04-26 04:58:55.276836991 +0000 UTC m=+0.018393712 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T04:58:55.696 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:55 smithi171 podman[106429]: 2024-04-26 04:58:55.393816324 +0000 UTC m=+0.135373055 container create 3a59de0dc94c98e1a3d7d68681e1bf2936e41ed501e58c9409117a4018173f08 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True) 2024-04-26T04:58:55.696 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:55 smithi171 podman[106429]: 2024-04-26 04:58:55.60444773 +0000 UTC m=+0.346004419 container init 3a59de0dc94c98e1a3d7d68681e1bf2936e41ed501e58c9409117a4018173f08 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T04:58:55.696 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:55 smithi171 podman[106429]: 2024-04-26 04:58:55.609765989 +0000 UTC m=+0.351322681 container start 3a59de0dc94c98e1a3d7d68681e1bf2936e41ed501e58c9409117a4018173f08 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T04:58:55.697 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:55 smithi171 podman[106429]: 2024-04-26 04:58:55.696040036 +0000 UTC m=+0.437596738 container attach 3a59de0dc94c98e1a3d7d68681e1bf2936e41ed501e58c9409117a4018173f08 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c) 2024-04-26T04:58:55.999 INFO:journalctl@ceph.osd.7.smithi171.stdout:Apr 26 04:58:55 smithi171 podman[106429]: 2024-04-26 04:58:55.891906265 +0000 UTC m=+0.633462960 container died 3a59de0dc94c98e1a3d7d68681e1bf2936e41ed501e58c9409117a4018173f08 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5) 2024-04-26T04:58:56.460 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@osd.7.service' 2024-04-26T04:58:56.497 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T04:58:56.497 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-26T04:58:56.498 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-26T04:58:56.498 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-110027f2-0386-11ef-bc93-c7b262605968@prometheus.a 2024-04-26T04:58:56.749 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:56 smithi171 systemd[1]: Stopping Ceph prometheus.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:58:57.128 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:56 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:58:56.846Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:58:57.128 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:56 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:58:56.847Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T04:58:57.128 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:56 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:58:56.847Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T04:58:57.128 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:56 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:58:56.847Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T04:58:57.128 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:56 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:58:56.847Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T04:58:57.129 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:56 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:58:56.847Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T04:58:57.129 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:56 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:58:56.847Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T04:58:57.129 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:56 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:58:56.847Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T04:58:57.129 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:56 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:58:56.847Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T04:58:57.129 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:56 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:58:56.847Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T04:58:57.129 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:56 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:58:56.847Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T04:58:57.129 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:56 smithi171 ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a[79500]: ts=2024-04-26T04:58:56.847Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T04:58:57.129 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:56 smithi171 podman[106559]: 2024-04-26 04:58:56.85718358 +0000 UTC m=+0.144669127 container died 93d6b22f6fcef0a3f36edba300841842ad87b33a57041fcd3f9d32679a67c9f7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:57.486 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:57 smithi171 podman[106559]: 2024-04-26 04:58:57.226788964 +0000 UTC m=+0.514274497 container cleanup 93d6b22f6fcef0a3f36edba300841842ad87b33a57041fcd3f9d32679a67c9f7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:57.486 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Apr 26 04:58:57 smithi171 bash[106559]: ceph-110027f2-0386-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T04:58:57.647 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-110027f2-0386-11ef-bc93-c7b262605968@prometheus.a.service' 2024-04-26T04:58:57.680 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T04:58:57.680 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-26T04:58:57.681 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 110027f2-0386-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-26T04:58:59.557 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:59 smithi156 systemd[1]: Stopping Ceph alertmanager.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:58:59.557 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:59 smithi156 ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a[88706]: ts=2024-04-26T04:58:59.505Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:58:59.820 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:58:59 smithi156 podman[154860]: 2024-04-26 04:58:59.556680302 +0000 UTC m=+0.153591089 container died 932345b7da326190269cf5d4e04632a5b73e593424331c04cd0b3df2dad8b12e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:00.092 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:59:00 smithi156 podman[154860]: 2024-04-26 04:59:00.019145233 +0000 UTC m=+0.616056028 container cleanup 932345b7da326190269cf5d4e04632a5b73e593424331c04cd0b3df2dad8b12e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:00.093 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:59:00 smithi156 bash[154860]: ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T04:59:00.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:59:00 smithi156 podman[154879]: 2024-04-26 04:59:00.287150396 +0000 UTC m=+0.726770232 container remove 932345b7da326190269cf5d4e04632a5b73e593424331c04cd0b3df2dad8b12e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:00.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:59:00 smithi156 podman[154879]: 2024-04-26 04:59:00.354463312 +0000 UTC m=+0.794083085 volume remove 7ef5b1e73ba0732ba8f2f2bad80d9d55e39038796eaabcdfd5d7858275d0dc3d 2024-04-26T04:59:00.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:59:00 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T04:59:00.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:59:00 smithi156 systemd[1]: Stopped Ceph alertmanager.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:59:00.593 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Apr 26 04:59:00 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 2.110s CPU time. 2024-04-26T04:59:01.093 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:59:00 smithi156 systemd[1]: Stopping Ceph node-exporter.a for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:59:01.093 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:59:00 smithi156 podman[154992]: 2024-04-26 04:59:00.920359246 +0000 UTC m=+0.103290997 container died 5af6998cc7e3ec4f95304ee91bae3375eceea078bc6b4e5886b9749459a5d1b7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:01.525 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:59:01 smithi156 podman[154992]: 2024-04-26 04:59:01.340828132 +0000 UTC m=+0.523776899 container cleanup 5af6998cc7e3ec4f95304ee91bae3375eceea078bc6b4e5886b9749459a5d1b7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:01.525 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:59:01 smithi156 bash[154992]: ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a 2024-04-26T04:59:01.780 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:59:01 smithi156 podman[155012]: 2024-04-26 04:59:01.525491045 +0000 UTC m=+0.609627688 container remove 5af6998cc7e3ec4f95304ee91bae3375eceea078bc6b4e5886b9749459a5d1b7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:59:01.780 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:59:01 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:59:01.780 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:59:01 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-26T04:59:01.780 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:59:01 smithi156 systemd[1]: Stopped Ceph node-exporter.a for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:59:01.781 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Apr 26 04:59:01 smithi156 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 2.596s CPU time. 2024-04-26T04:59:13.220 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 110027f2-0386-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-26T04:59:15.219 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:59:14 smithi171 systemd[1]: Stopping Ceph node-exporter.b for 110027f2-0386-11ef-bc93-c7b262605968... 2024-04-26T04:59:15.499 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:59:15 smithi171 podman[107035]: 2024-04-26 04:59:15.218257892 +0000 UTC m=+0.126888039 container died a2d37cfca8cf7132ee7252fe19648f08ec8ff7fe47ebf6a819fc178a7f8340e7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:59:15.830 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:59:15 smithi171 podman[107035]: 2024-04-26 04:59:15.629561719 +0000 UTC m=+0.538191897 container cleanup a2d37cfca8cf7132ee7252fe19648f08ec8ff7fe47ebf6a819fc178a7f8340e7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:59:15.831 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:59:15 smithi171 bash[107035]: ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b 2024-04-26T04:59:16.234 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:59:15 smithi171 podman[107054]: 2024-04-26 04:59:15.830399169 +0000 UTC m=+0.616718759 container remove a2d37cfca8cf7132ee7252fe19648f08ec8ff7fe47ebf6a819fc178a7f8340e7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-110027f2-0386-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:59:16.234 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:59:15 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:59:16.234 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:59:16 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-26T04:59:16.234 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:59:16 smithi171 systemd[1]: Stopped Ceph node-exporter.b for 110027f2-0386-11ef-bc93-c7b262605968. 2024-04-26T04:59:16.234 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Apr 26 04:59:16 smithi171 systemd[1]: ceph-110027f2-0386-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 2.517s CPU time. 2024-04-26T04:59:21.888 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:21.920 DEBUG:teuthology.orchestra.run.smithi171:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:21.945 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-26T04:59:21.947 DEBUG:teuthology.misc:Transferring archived files from smithi156:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674565/remote/smithi156/crash 2024-04-26T04:59:21.948 DEBUG:teuthology.orchestra.run.smithi156:> sudo tar c -f - -C /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/crash -- . 2024-04-26T04:59:22.032 INFO:teuthology.orchestra.run.smithi156.stderr:tar: /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-26T04:59:22.032 INFO:teuthology.orchestra.run.smithi156.stderr:tar: Error is not recoverable: exiting now 2024-04-26T04:59:22.034 DEBUG:teuthology.misc:Transferring archived files from smithi171:/var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674565/remote/smithi171/crash 2024-04-26T04:59:22.035 DEBUG:teuthology.orchestra.run.smithi171:> sudo tar c -f - -C /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/crash -- . 2024-04-26T04:59:22.065 INFO:teuthology.orchestra.run.smithi171.stderr:tar: /var/lib/ceph/110027f2-0386-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-26T04:59:22.065 INFO:teuthology.orchestra.run.smithi171.stderr:tar: Error is not recoverable: exiting now 2024-04-26T04:59:22.066 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-26T04:59:22.066 DEBUG:teuthology.orchestra.run.smithi156:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-26T04:59:22.094 INFO:teuthology.orchestra.run.smithi156.stdout:1714106287.6021707 mon.a (mon.0) 502 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T04:59:22.094 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-26T04:59:22.095 INFO:tasks.cephadm:Compressing logs... 2024-04-26T04:59:22.095 DEBUG:teuthology.orchestra.run.smithi156:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-26T04:59:22.137 DEBUG:teuthology.orchestra.run.smithi171:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-26T04:59:22.158 INFO:teuthology.orchestra.run.smithi156.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mds.foo.smithi156.wyfzii.log 2024-04-26T04:59:22.159 INFO:teuthology.orchestra.run.smithi156.stderr:: No such file or directory 2024-04-26T04:59:22.174 INFO:teuthology.orchestra.run.smithi171.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.log 2024-04-26T04:59:22.174 INFO:teuthology.orchestra.run.smithi171.stderr:: No such file or directory 2024-04-26T04:59:22.174 INFO:teuthology.orchestra.run.smithi171.stderr:gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-26T04:59:22.174 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.7.log 2024-04-26T04:59:22.174 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.5.log 2024-04-26T04:59:22.174 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mds.foo.smithi171.kzjlqn.log 2024-04-26T04:59:22.175 INFO:teuthology.orchestra.run.smithi171.stderr:gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.4.log 2024-04-26T04:59:22.175 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.5.log: /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mds.foo.smithi171.kzjlqn.log: gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mgr.x.log 2024-04-26T04:59:22.175 INFO:teuthology.orchestra.run.smithi171.stderr: 62.3% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mds.foo.smithi171.kzjlqn.log.gz 2024-04-26T04:59:22.175 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.6.log 2024-04-26T04:59:22.175 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-26T04:59:22.175 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-26T04:59:22.175 INFO:teuthology.orchestra.run.smithi171.stderr: 87.7% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-26T04:59:22.175 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mon.b.log 2024-04-26T04:59:22.176 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-volume.log: 83.6% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-26T04:59:22.176 INFO:teuthology.orchestra.run.smithi171.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-26T04:59:22.176 INFO:teuthology.orchestra.run.smithi171.stderr: 89.5% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-26T04:59:22.176 INFO:teuthology.orchestra.run.smithi156.stderr:gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.log 2024-04-26T04:59:22.177 INFO:teuthology.orchestra.run.smithi156.stderr:gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.2.log 2024-04-26T04:59:22.177 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mds.foo.smithi156.wyfzii.log: /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-26T04:59:22.177 INFO:teuthology.orchestra.run.smithi156.stderr: 81.8% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mds.foo.smithi156.wyfzii.log.gz 2024-04-26T04:59:22.177 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.3.log 2024-04-26T04:59:22.177 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.1.log 2024-04-26T04:59:22.177 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mgr.y.log 2024-04-26T04:59:22.178 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.0.log 2024-04-26T04:59:22.178 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mon.c.log 2024-04-26T04:59:22.178 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mon.a.log 2024-04-26T04:59:22.178 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mon.c.log: 92.8% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-26T04:59:22.178 INFO:teuthology.orchestra.run.smithi156.stderr:gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-26T04:59:22.178 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mon.a.log: 93.6% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-26T04:59:22.179 INFO:teuthology.orchestra.run.smithi156.stderr:gzip -5 --verbose -- /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-26T04:59:22.179 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.cephadm.log: 90.2% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-26T04:59:22.179 INFO:teuthology.orchestra.run.smithi156.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-26T04:59:22.180 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-26T04:59:22.187 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-volume.log: /var/log/ceph/cephadm.log: 93.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-26T04:59:22.189 INFO:teuthology.orchestra.run.smithi156.stderr: 91.1% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-26T04:59:22.191 INFO:teuthology.orchestra.run.smithi171.stderr: 91.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-26T04:59:22.227 INFO:teuthology.orchestra.run.smithi171.stderr: 89.7% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mgr.x.log.gz 2024-04-26T04:59:22.463 INFO:teuthology.orchestra.run.smithi171.stderr: 92.4% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mon.b.log.gz 2024-04-26T04:59:22.521 INFO:teuthology.orchestra.run.smithi156.stderr: 88.6% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mgr.y.log.gz 2024-04-26T04:59:22.535 INFO:teuthology.orchestra.run.smithi156.stderr: 92.1% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mon.c.log.gz 2024-04-26T04:59:22.704 INFO:teuthology.orchestra.run.smithi171.stderr: 92.9% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.6.log.gz 2024-04-26T04:59:22.713 INFO:teuthology.orchestra.run.smithi171.stderr: 93.6% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.5.log.gz 2024-04-26T04:59:22.750 INFO:teuthology.orchestra.run.smithi171.stderr: 93.6% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.4.log.gz 2024-04-26T04:59:22.801 INFO:teuthology.orchestra.run.smithi156.stderr: 93.3% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.1.log.gz 2024-04-26T04:59:22.822 INFO:teuthology.orchestra.run.smithi156.stderr: 93.4% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.3.log.gz 2024-04-26T04:59:22.930 INFO:teuthology.orchestra.run.smithi171.stderr: 93.8% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.7.log.gz 2024-04-26T04:59:22.931 INFO:teuthology.orchestra.run.smithi156.stderr: 93.2% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.0.log.gz 2024-04-26T04:59:22.932 INFO:teuthology.orchestra.run.smithi171.stderr: 2024-04-26T04:59:22.932 INFO:teuthology.orchestra.run.smithi171.stderr:real 0m0.781s 2024-04-26T04:59:22.932 INFO:teuthology.orchestra.run.smithi171.stderr:user 0m2.747s 2024-04-26T04:59:22.933 INFO:teuthology.orchestra.run.smithi171.stderr:sys 0m0.132s 2024-04-26T04:59:23.141 INFO:teuthology.orchestra.run.smithi156.stderr: 94.9% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-osd.2.log.gz 2024-04-26T04:59:23.244 INFO:teuthology.orchestra.run.smithi156.stderr: 90.6% -- replaced with /var/log/ceph/110027f2-0386-11ef-bc93-c7b262605968/ceph-mon.a.log.gz 2024-04-26T04:59:23.246 INFO:teuthology.orchestra.run.smithi156.stderr: 2024-04-26T04:59:23.246 INFO:teuthology.orchestra.run.smithi156.stderr:real 0m1.097s 2024-04-26T04:59:23.247 INFO:teuthology.orchestra.run.smithi156.stderr:user 0m4.731s 2024-04-26T04:59:23.247 INFO:teuthology.orchestra.run.smithi156.stderr:sys 0m0.182s 2024-04-26T04:59:23.247 INFO:tasks.cephadm:Archiving logs... 2024-04-26T04:59:23.247 DEBUG:teuthology.misc:Transferring archived files from smithi156:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674565/remote/smithi156/log 2024-04-26T04:59:23.248 DEBUG:teuthology.orchestra.run.smithi156:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-26T04:59:23.953 DEBUG:teuthology.misc:Transferring archived files from smithi171:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674565/remote/smithi171/log 2024-04-26T04:59:23.954 DEBUG:teuthology.orchestra.run.smithi171:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-26T04:59:24.348 INFO:tasks.cephadm:Removing cluster... 2024-04-26T04:59:24.348 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 110027f2-0386-11ef-bc93-c7b262605968 --force 2024-04-26T04:59:24.824 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 110027f2-0386-11ef-bc93-c7b262605968 --force 2024-04-26T04:59:25.312 INFO:tasks.cephadm:Removing cephadm ... 2024-04-26T04:59:25.313 DEBUG:teuthology.orchestra.run.smithi156:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-26T04:59:25.328 DEBUG:teuthology.orchestra.run.smithi171:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-26T04:59:25.343 INFO:tasks.cephadm:Teardown complete 2024-04-26T04:59:25.344 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-04-26T04:59:25.355 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-04-26T04:59:25.356 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-26T04:59:25.371 DEBUG:teuthology.orchestra.run.smithi171:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-26T04:59:25.454 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-26T04:59:25.463 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-26T04:59:25.463 DEBUG:teuthology.orchestra.run.smithi156:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T04:59:25.466 DEBUG:teuthology.orchestra.run.smithi171:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T04:59:25.477 INFO:teuthology.orchestra.run.smithi156.stderr:bash: line 1: ntpq: command not found 2024-04-26T04:59:25.479 INFO:teuthology.orchestra.run.smithi171.stderr:bash: line 1: ntpq: command not found 2024-04-26T04:59:25.482 INFO:teuthology.orchestra.run.smithi156.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T04:59:25.482 INFO:teuthology.orchestra.run.smithi156.stdout:=============================================================================== 2024-04-26T04:59:25.483 INFO:teuthology.orchestra.run.smithi156.stdout:^+ hv01.front.sepia.ceph.com 3 6 377 39 +1149us[+1149us] +/- 43ms 2024-04-26T04:59:25.483 INFO:teuthology.orchestra.run.smithi156.stdout:^* hv02.front.sepia.ceph.com 3 7 377 102 -1288us[-1298us] +/- 62ms 2024-04-26T04:59:25.483 INFO:teuthology.orchestra.run.smithi156.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 35 -83us[ -83us] +/- 73ms 2024-04-26T04:59:25.483 INFO:teuthology.orchestra.run.smithi156.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:59:25.484 INFO:teuthology.orchestra.run.smithi171.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T04:59:25.484 INFO:teuthology.orchestra.run.smithi171.stdout:=============================================================================== 2024-04-26T04:59:25.484 INFO:teuthology.orchestra.run.smithi171.stdout:^* hv01.front.sepia.ceph.com 3 6 377 33 +1051us[+1051us] +/- 43ms 2024-04-26T04:59:25.484 INFO:teuthology.orchestra.run.smithi171.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 35 -1396us[-1356us] +/- 63ms 2024-04-26T04:59:25.484 INFO:teuthology.orchestra.run.smithi171.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 36 -161us[ -121us] +/- 73ms 2024-04-26T04:59:25.484 INFO:teuthology.orchestra.run.smithi171.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:59:25.485 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-26T04:59:25.494 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-26T04:59:25.495 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-26T04:59:25.506 DEBUG:teuthology.orchestra.run.smithi156:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-26T04:59:25.611 DEBUG:teuthology.orchestra.run.smithi171:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-26T04:59:25.685 DEBUG:teuthology.orchestra.run.smithi156:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T04:59:25.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:59:25.713 DEBUG:teuthology.orchestra.run.smithi171:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T04:59:25.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:59:25.741 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-26T04:59:25.751 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-26T04:59:25.760 INFO:teuthology.task.internal:Duration was 1936.372958 seconds 2024-04-26T04:59:25.760 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-26T04:59:25.769 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-26T04:59:25.770 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-26T04:59:25.772 DEBUG:teuthology.orchestra.run.smithi171:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-26T04:59:25.806 INFO:teuthology.orchestra.run.smithi156.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T04:59:25.817 INFO:teuthology.orchestra.run.smithi171.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T04:59:26.126 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-26T04:59:26.126 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi156.front.sepia.ceph.com 2024-04-26T04:59:26.127 DEBUG:teuthology.orchestra.run.smithi156:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-26T04:59:26.186 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi171.front.sepia.ceph.com 2024-04-26T04:59:26.186 DEBUG:teuthology.orchestra.run.smithi171:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-26T04:59:26.219 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-26T04:59:26.219 DEBUG:teuthology.orchestra.run.smithi156:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-26T04:59:26.228 DEBUG:teuthology.orchestra.run.smithi171:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-26T04:59:26.290 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-26T04:59:26.290 DEBUG:teuthology.orchestra.run.smithi156:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-26T04:59:26.295 DEBUG:teuthology.orchestra.run.smithi171:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-26T04:59:26.710 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-26T04:59:26.721 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-26T04:59:26.721 DEBUG:teuthology.orchestra.run.smithi156:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-26T04:59:26.745 DEBUG:teuthology.orchestra.run.smithi171:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-26T04:59:26.769 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-26T04:59:26.780 DEBUG:teuthology.orchestra.run.smithi156:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-26T04:59:26.788 DEBUG:teuthology.orchestra.run.smithi171:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-26T04:59:26.807 INFO:teuthology.orchestra.run.smithi156.stdout:kernel.core_pattern = core 2024-04-26T04:59:26.831 INFO:teuthology.orchestra.run.smithi171.stdout:kernel.core_pattern = core 2024-04-26T04:59:26.862 DEBUG:teuthology.orchestra.run.smithi156:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-26T04:59:26.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:59:26.876 DEBUG:teuthology.orchestra.run.smithi171:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-26T04:59:26.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:59:26.919 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-26T04:59:26.930 INFO:teuthology.task.internal:Transferring archived files... 2024-04-26T04:59:26.931 DEBUG:teuthology.misc:Transferring archived files from smithi156:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674565/remote/smithi156 2024-04-26T04:59:26.931 DEBUG:teuthology.orchestra.run.smithi156:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-26T04:59:26.977 DEBUG:teuthology.misc:Transferring archived files from smithi171:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674565/remote/smithi171 2024-04-26T04:59:26.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-26T04:59:27.013 INFO:teuthology.task.internal:Removing archive directory... 2024-04-26T04:59:27.013 DEBUG:teuthology.orchestra.run.smithi156:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-26T04:59:27.015 DEBUG:teuthology.orchestra.run.smithi171:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-26T04:59:27.062 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-26T04:59:27.075 INFO:teuthology.task.internal:Not uploading archives. 2024-04-26T04:59:27.075 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-26T04:59:27.087 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-26T04:59:27.087 DEBUG:teuthology.orchestra.run.smithi156:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-26T04:59:27.090 DEBUG:teuthology.orchestra.run.smithi171:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-26T04:59:27.101 INFO:teuthology.orchestra.run.smithi156.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 26 04:59 /home/ubuntu/cephtest 2024-04-26T04:59:27.115 INFO:teuthology.orchestra.run.smithi171.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 26 04:59 /home/ubuntu/cephtest 2024-04-26T04:59:27.117 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-26T04:59:27.128 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-26T04:59:27.150 INFO:teuthology.nuke:Checking targets against current locks 2024-04-26T04:59:27.178 DEBUG:teuthology.nuke:shortname: smithi156 2024-04-26T04:59:27.178 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T04:59:27.201 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi156.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674565', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:21:28.743832', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM2HXQRkDls+lC4fcS46mnWL3XBA8xna/tyo6rjNsgsOoG+yq0T9ZYBIUPBdqThsWnNy9o2Um+r879jCRbZGeHI='} 2024-04-26T04:59:27.204 DEBUG:teuthology.nuke:shortname: smithi171 2024-04-26T04:59:27.205 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T04:59:27.227 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi171.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674565', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:21:28.745736', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGcPaoiQ6uEEM2skH/wzALXnaue1QlScxv574JI4U9eglkXMB8nyMxVbjl/VlSOv1IF1UwMvXMMWK8+ACxW7Ud0='} 2024-04-26T04:59:27.256 INFO:teuthology.orchestra.console.smithi156:Power off 2024-04-26T04:59:27.256 DEBUG:teuthology.orchestra.console.smithi156:pexpect command: ipmitool -H smithi156.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-26T04:59:27.280 DEBUG:teuthology.orchestra.console.smithi156:power off output: Chassis Power Control: Down/Off 2024-04-26T04:59:27.281 DEBUG:teuthology.orchestra.console.smithi156:pexpect command: ipmitool -H smithi156.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T04:59:27.296 INFO:teuthology.orchestra.console.smithi171:Power off 2024-04-26T04:59:27.297 DEBUG:teuthology.orchestra.console.smithi171:pexpect command: ipmitool -H smithi171.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-26T04:59:27.305 DEBUG:teuthology.orchestra.console.smithi156:check power output: Chassis Power is on 2024-04-26T04:59:27.320 DEBUG:teuthology.orchestra.console.smithi171:power off output: Chassis Power Control: Down/Off 2024-04-26T04:59:27.320 DEBUG:teuthology.orchestra.console.smithi171:pexpect command: ipmitool -H smithi171.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T04:59:27.342 DEBUG:teuthology.orchestra.console.smithi171:check power output: Chassis Power is on 2024-04-26T04:59:31.306 DEBUG:teuthology.orchestra.console.smithi156:pexpect command: ipmitool -H smithi156.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T04:59:31.343 DEBUG:teuthology.orchestra.console.smithi171:pexpect command: ipmitool -H smithi171.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T04:59:31.416 DEBUG:teuthology.orchestra.console.smithi156:check power output: Chassis Power is on 2024-04-26T04:59:31.453 DEBUG:teuthology.orchestra.console.smithi171:check power output: Chassis Power is on 2024-04-26T04:59:35.418 DEBUG:teuthology.orchestra.console.smithi156:pexpect command: ipmitool -H smithi156.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T04:59:35.454 DEBUG:teuthology.orchestra.console.smithi171:pexpect command: ipmitool -H smithi171.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T04:59:35.530 DEBUG:teuthology.orchestra.console.smithi156:check power output: Chassis Power is off 2024-04-26T04:59:35.564 DEBUG:teuthology.orchestra.console.smithi171:check power output: Chassis Power is on 2024-04-26T04:59:35.632 INFO:teuthology.orchestra.console.smithi156:Power off completed 2024-04-26T04:59:39.569 DEBUG:teuthology.orchestra.console.smithi171:pexpect command: ipmitool -H smithi171.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T04:59:39.680 DEBUG:teuthology.orchestra.console.smithi171:check power output: Chassis Power is off 2024-04-26T04:59:39.780 INFO:teuthology.orchestra.console.smithi171:Power off completed 2024-04-26T04:59:39.885 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} duration: 1936.3729584217072 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=7dfbfe6bc7c74b668350e5c08a4a6543 status: fail success: false 2024-04-26T04:59:39.885 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-26T04:59:39.965 INFO:teuthology.run:FAIL